Reading variables from file

acceptance/bundle/variables/empty/output.txt

@@ -1,4 +1,4 @@
-Error: no value assigned to required variable a. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_a environment variable
+Error: no value assigned to required variable a. Assignment can be done using "--var", by setting the BUNDLE_VAR_a environment variable, or in .databricks/bundle/<target>/variable-overrides.json file
 
 Name: empty${var.a}
 Target: default

acceptance/bundle/variables/env_overrides/output.txt

@@ -9,7 +9,7 @@
 "prod-a env-var-b"
 
 >>> errcode $CLI bundle validate -t env-missing-a-required-variable-assignment
-Error: no value assigned to required variable b. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_b environment variable
+Error: no value assigned to required variable b. Assignment can be done using "--var", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle/<target>/variable-overrides.json file
 
 Name: test bundle
 Target: env-missing-a-required-variable-assignment

acceptance/bundle/variables/file-defaults/.databricks/bundle/complex_to_string/variable-overrides.json

@@ -0,0 +1,5 @@
+{
+ "cluster_key": {
+ "node_type_id": "Standard_DS3_v2"
+ }
+}

acceptance/bundle/variables/file-defaults/.databricks/bundle/default/variable-overrides.json

@@ -0,0 +1,7 @@
+{
+ "cluster": {
+ "node_type_id": "Standard_DS3_v2"
+ },
+ "cluster_key": "mlops_stacks-cluster",
+ "cluster_workers": 2
+}

acceptance/bundle/variables/file-defaults/.databricks/bundle/invalid_json/variable-overrides.json

@@ -0,0 +1 @@
+foo

acceptance/bundle/variables/file-defaults/.databricks/bundle/string_to_complex/variable-overrides.json

@@ -0,0 +1,3 @@
+{
+ "cluster": "mlops_stacks-cluster"
+}

acceptance/bundle/variables/file-defaults/.databricks/bundle/with_value/variable-overrides.json

@@ -0,0 +1,3 @@
+{
+ "cluster_key": "mlops_stacks-cluster-from-file"
+}

acceptance/bundle/variables/file-defaults/.databricks/bundle/without_defaults/variable-overrides.json

@@ -0,0 +1,4 @@
+{
+ "cluster_key": "mlops_stacks-cluster",
+ "cluster_workers": 2
+}

acceptance/bundle/variables/file-defaults/.databricks/bundle/wrong_file_structure/variable-overrides.json

@@ -0,0 +1,3 @@
+[
+ "foo"
+]

acceptance/bundle/variables/file-defaults/.gitignore

@@ -0,0 +1 @@
+!.databricks

acceptance/bundle/variables/file-defaults/databricks.yml

@@ -0,0 +1,53 @@
+bundle:
+ name: TestResolveVariablesFromFile
+
+variables:
+ cluster:
+ type: "complex"
+ cluster_key:
+ cluster_workers:
+
+resources:
+ jobs:
+ job1:
+ job_clusters:
+ - job_cluster_key: ${var.cluster_key}
+ new_cluster:
+ node_type_id: "${var.cluster.node_type_id}"
+ num_workers: ${var.cluster_workers}
+
+targets:
+ default:
+ default: true
+ variables:
+ cluster_workers: 1
+ cluster:
+ node_type_id: "default"
+ cluster_key: "default"
+
+ without_defaults:
+
+ complex_to_string:
+ variables:
+ cluster_workers: 1
+ cluster:
+ node_type_id: "default"
+ cluster_key: "default"
+
+ string_to_complex:
+ variables:
+ cluster_workers: 1
+ cluster:
+ node_type_id: "default"
+ cluster_key: "default"
+
+ wrong_file_structure:
+
+ invalid_json:
+
+ with_value:
+ variables:
+ cluster_workers: 1
+ cluster:
+ node_type_id: "default"
+ cluster_key: cluster_key_value

acceptance/bundle/variables/file-defaults/output.txt

@@ -0,0 +1,82 @@
+
+=== variable file
+>>> $CLI bundle validate -o json
+{
+ "job_cluster_key": "mlops_stacks-cluster",
+ "new_cluster": {
+ "node_type_id": "Standard_DS3_v2",
+ "num_workers": 2
+ }
+}
+
+=== variable file and variable flag
+>>> $CLI bundle validate -o json --var=cluster_key=mlops_stacks-cluster-overriden
+{
+ "job_cluster_key": "mlops_stacks-cluster-overriden",
+ "new_cluster": {
+ "node_type_id": "Standard_DS3_v2",
+ "num_workers": 2
+ }
+}
+
+=== variable file and environment variable
+>>> BUNDLE_VAR_cluster_key=mlops_stacks-cluster-overriden $CLI bundle validate -o json
+{
+ "job_cluster_key": "mlops_stacks-cluster-overriden",
+ "new_cluster": {
+ "node_type_id": "Standard_DS3_v2",
+ "num_workers": 2
+ }
+}
+
+=== variable has value in config file
+>>> $CLI bundle validate -o json --target with_value
+{
+ "job_cluster_key": "mlops_stacks-cluster-from-file",
+ "new_cluster": {
+ "node_type_id": "default",
+ "num_workers": 1
+ }
+}
+
+=== file has variable that is complex but default is string
+>>> errcode $CLI bundle validate -o json --target complex_to_string
+Error: variable cluster_key is not of type complex, but the value in the variable file is a complex type
+
+
+Exit code: 1
+{
+ "job_cluster_key": "${var.cluster_key}",
+ "new_cluster": {
+ "node_type_id": "${var.cluster.node_type_id}",
+ "num_workers": "${var.cluster_workers}"
+ }
+}
+
+=== file has variable that is string but default is complex
+>>> errcode $CLI bundle validate -o json --target string_to_complex
+Error: variable cluster is of type complex, but the value in the variable file is not a complex type
+
+
+Exit code: 1
+{
+ "job_cluster_key": "${var.cluster_key}",
+ "new_cluster": {
+ "node_type_id": "${var.cluster.node_type_id}",
+ "num_workers": "${var.cluster_workers}"
+ }
+}
+
+=== variable is required but it's not provided in the file
+>>> errcode $CLI bundle validate -o json --target without_defaults
+Error: no value assigned to required variable cluster. Assignment can be done using "--var", by setting the BUNDLE_VAR_cluster environment variable, or in .databricks/bundle/<target>/variable-overrides.json file
+
+
+Exit code: 1
+{
+ "job_cluster_key": "${var.cluster_key}",
+ "new_cluster": {
+ "node_type_id": "${var.cluster.node_type_id}",
+ "num_workers": "${var.cluster_workers}"
+ }
+}

acceptance/bundle/variables/file-defaults/script

@@ -0,0 +1,30 @@
+cluster_expr=".resources.jobs.job1.job_clusters[0]"
+
+# defaults from variable file, see .databricks/bundle/<target>/variable-overrides.json
+
+title "variable file"
+trace $CLI bundle validate -o json | jq $cluster_expr
+
+title "variable file and variable flag"
+trace $CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster-overriden" | jq $cluster_expr
+
+title "variable file and environment variable"
+trace BUNDLE_VAR_cluster_key=mlops_stacks-cluster-overriden $CLI bundle validate -o json | jq $cluster_expr
+
+title "variable has value in config file"
+trace $CLI bundle validate -o json --target with_value | jq $cluster_expr
+
+# title "file cannot be parsed"
+# trace errcode $CLI bundle validate -o json --target invalid_json | jq $cluster_expr
+
+# title "file has wrong structure"
+# trace errcode $CLI bundle validate -o json --target wrong_file_structure | jq $cluster_expr
+
+title "file has variable that is complex but default is string"
+trace errcode $CLI bundle validate -o json --target complex_to_string | jq $cluster_expr
+
+title "file has variable that is string but default is complex"
+trace errcode $CLI bundle validate -o json --target string_to_complex | jq $cluster_expr
+
+title "variable is required but it's not provided in the file"
+trace errcode $CLI bundle validate -o json --target without_defaults | jq $cluster_expr

acceptance/bundle/variables/vanilla/output.txt

@@ -3,7 +3,7 @@
 "abc def"
 
 >>> errcode $CLI bundle validate
-Error: no value assigned to required variable b. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_b environment variable
+Error: no value assigned to required variable b. Assignment can be done using "--var", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle/<target>/variable-overrides.json file
 
 Name: ${var.a} ${var.b}
 Target: default

acceptance/script.prepare

@@ -40,3 +40,8 @@ git-repo-init() {
  git add databricks.yml
  git commit -qm 'Add databricks.yml'
 }
+
+title() {
+ local label="$1"
+ printf "\n=== %s" "$label"
+}

bundle/config/mutator/set_variables.go

@@ -3,11 +3,14 @@ package mutator
 import (
 	"context"
 	"fmt"
+	"os"
+	"path/filepath"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config/variable"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/jsonloader"
 	"github.com/databricks/cli/libs/env"
 )
 
@@ -23,7 +26,11 @@ func (m *setVariables) Name() string {
 	return "SetVariables"
 }
 
-func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, name string) (dyn.Value, error) {
+func getDefaultVariableFilePath(target string) string {
+	return ".databricks/bundle/" + target + "/variable-overrides.json"
+}
+
+func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, name string, fileDefault dyn.Value) (dyn.Value, error) {
 	// case: variable already has value initialized, so skip
 	if variable.HasValue() {
  return v, nil
@@ -49,6 +56,26 @@ func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable,
  return v, nil
 	}
 
+	// case: Set the variable to the default value from the variable file
+	if fileDefault.Kind() != dyn.KindInvalid && fileDefault.Kind() != dyn.KindNil {
+ hasComplexType := variable.IsComplex()
+ hasComplexValue := fileDefault.Kind() == dyn.KindMap || fileDefault.Kind() == dyn.KindSequence
+
+ if hasComplexType && !hasComplexValue {
+ return dyn.InvalidValue, fmt.Errorf(`variable %s is of type complex, but the value in the variable file is not a complex type`, name)
+ }
+ if !hasComplexType && hasComplexValue {
+ return dyn.InvalidValue, fmt.Errorf(`variable %s is not of type complex, but the value in the variable file is a complex type`, name)
+ }
+
+ v, err := dyn.Set(v, "value", fileDefault)
+ if err != nil {
+ return dyn.InvalidValue, fmt.Errorf(`failed to assign default value from variable file to variable %s with error: %v`, name, err)
+ }
+
+ return v, nil
+	}
+
 	// case: Set the variable to its default value
 	if variable.HasDefault() {
  vDefault, err := dyn.Get(v, "default")
@@ -64,10 +91,43 @@ func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable,
 	}
 
 	// We should have had a value to set for the variable at this point.
-	return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" flag or by setting the %s environment variable`, name, bundleVarPrefix+name)
+	return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done using "--var", by setting the %s environment variable, or in %s file`, name, bundleVarPrefix+name, getDefaultVariableFilePath("<target>"))
+}
+
+func readVariablesFromFile(b *bundle.Bundle) (dyn.Value, diag.Diagnostics) {
+	var diags diag.Diagnostics
+
+	filePath := filepath.Join(b.BundleRootPath, getDefaultVariableFilePath(b.Config.Bundle.Target))
+	if _, err := os.Stat(filePath); err != nil {
+ return dyn.InvalidValue, nil
+	}
+
+	f, err := os.ReadFile(filePath)
+	if err != nil {
+ return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to read variables file: %w", err))
+	}
+
+	val, err := jsonloader.LoadJSON(f, filePath)
+	if err != nil {
+ return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to parse variables file %s: %w", filePath, err))
+	}
+
+	if val.Kind() != dyn.KindMap {
+ return dyn.InvalidValue, diags.Append(diag.Diagnostic{
+ Severity: diag.Error,
+ Summary: fmt.Sprintf("failed to parse variables file %s: invalid format", filePath),
+ Detail: "Variables file must be a JSON object with the following format:\n{\"var1\": \"value1\", \"var2\": \"value2\"}",
+ })
+	}
+
+	return val, nil
 }
 
 func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	defaults, diags := readVariablesFromFile(b)
+	if diags.HasError() {
+ return diags
+	}
 	err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
  return dyn.Map(v, "variables", dyn.Foreach(func(p dyn.Path, variable dyn.Value) (dyn.Value, error) {
  name := p[1].Key()
@@ -76,9 +136,10 @@ func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
  return dyn.InvalidValue, fmt.Errorf(`variable "%s" is not defined`, name)
  }
 
- return setVariable(ctx, variable, v, name)
+ fileDefault, _ := dyn.Get(defaults, name)
+ return setVariable(ctx, variable, v, name, fileDefault)
  }))
 	})
 
-	return diag.FromErr(err)
+	return diags.Extend(diag.FromErr(err))
 }