Add env section to bundle scripts for DABs interpolation

NEXT_CHANGELOG.md

@@ -8,6 +8,8 @@
 
 ### Bundles
 
+* Added an `env:` section to `scripts.<name>` for declaring environment variables that may reference `${bundle.*}`, `${workspace.*}`, and `${var.*}` ([#4179](https://github.com/databricks/cli/issues/4179)).
+
 ### Dependency updates
 
 ### API Changes

acceptance/bundle/run/scripts/env-bad-prefix/databricks.yml

@@ -0,0 +1,14 @@
+bundle:
+ name: script-env-bad-prefix
+
+resources:
+ jobs:
+ my_job:
+ name: my-job
+
+scripts:
+ bad:
+ env:
+ JOB_ID: ${resources.jobs.my_job.id}
+ UNKNOWN: ${something.else}
+ content: echo "$JOB_ID $UNKNOWN"

acceptance/bundle/run/scripts/env-bad-prefix/output.txt

@@ -0,0 +1,14 @@
+
+>>> [CLI] bundle validate
+Error: ${resources.jobs.my_job.id} cannot be used in scripts.bad.env.JOB_ID; only ${bundle.*}, ${workspace.*}, and ${var.*} are resolved before scripts execute
+ at scripts.bad.env.JOB_ID
+ in databricks.yml:12:15
+
+Error: ${something.else} cannot be used in scripts.bad.env.UNKNOWN; only ${bundle.*}, ${workspace.*}, and ${var.*} are resolved before scripts execute
+ at scripts.bad.env.UNKNOWN
+ in databricks.yml:13:16
+
+Name: script-env-bad-prefix
+Target: default
+
+Found 2 errors

acceptance/bundle/run/scripts/env-bad-prefix/script

@@ -0,0 +1 @@
+musterr trace $CLI bundle validate

acceptance/bundle/run/scripts/env-bad-prefix/test.toml

@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"]

acceptance/bundle/run/scripts/env-precedence/databricks.yml

@@ -0,0 +1,14 @@
+bundle:
+ name: script-env-precedence
+
+targets:
+ default:
+ default: true
+
+# A script's env: entry must win over the DATABRICKS_BUNDLE_TARGET variable the
+# CLI injects into the script process (executeScript appends env: last).
+scripts:
+ show_target:
+ env:
+ DATABRICKS_BUNDLE_TARGET: from-script-env
+ content: echo "target=$DATABRICKS_BUNDLE_TARGET"

acceptance/bundle/run/scripts/env-precedence/output.txt

@@ -0,0 +1,3 @@
+
+>>> [CLI] bundle run show_target
+target=from-script-env

acceptance/bundle/run/scripts/env-precedence/script

@@ -0,0 +1 @@
+trace $CLI bundle run show_target

acceptance/bundle/run/scripts/env-precedence/test.toml

@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"]

acceptance/bundle/run/scripts/env-section/databricks.yml

@@ -0,0 +1,21 @@
+bundle:
+ name: script-env-section
+
+variables:
+ region:
+ default: us-west-2
+ database:
+ default: mydb
+
+scripts:
+ show_env:
+ env:
+ BUNDLE_NAME: ${bundle.name}
+ REGION: ${var.region}
+ DATABASE: ${var.database}
+ MIXED: "region=${var.region};db=${var.database}"
+ content: |-
+ echo "bundle=$BUNDLE_NAME"
+ echo "region=$REGION"
+ echo "database=$DATABASE"
+ echo "mixed=$MIXED"

acceptance/bundle/run/scripts/env-section/output.txt

@@ -0,0 +1,6 @@
+
+>>> [CLI] bundle run show_env
+bundle=script-env-section
+region=us-west-2
+database=mydb
+mixed=region=us-west-2;db=mydb

acceptance/bundle/run/scripts/env-section/script

@@ -0,0 +1 @@
+trace $CLI bundle run show_env

acceptance/bundle/run/scripts/env-section/test.toml

@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"]

acceptance/bundle/run/scripts/no-interpolation/output.txt

@@ -1,40 +1,64 @@
 
 >>> [CLI] bundle deploy
-Error: Found ${FOO} in script one. Interpolation syntax ${...} is not allowed in scripts
+Error: Found ${FOO} in script one.content. Interpolation syntax ${...} is not supported in script content
  at scripts.one.content
  in databricks.yml:6:14
 
-We do not support the ${...} interpolation syntax in scripts because
-it's ambiguous whether it's a variable reference or reference to an
-environment variable.
+The ${...} syntax is not allowed in script content. To use a bundle value,
+declare an environment variable in the script's "env:" section and reference it
+from "content" with $NAME:
 
-Error: Found ${var.BAR} in script two. Interpolation syntax ${...} is not allowed in scripts
+ scripts:
+ one:
+ env:
+ MY_VAR: ${var.foo}
+ content: echo "$MY_VAR"
+
+Error: Found ${var.BAR} in script two.content. Interpolation syntax ${...} is not supported in script content
  at scripts.two.content
  in databricks.yml:8:14
 
-We do not support the ${...} interpolation syntax in scripts because
-it's ambiguous whether it's a variable reference or reference to an
-environment variable.
+The ${...} syntax is not allowed in script content. To use a bundle value,
+declare an environment variable in the script's "env:" section and reference it
+from "content" with $NAME:
+
+ scripts:
+ two:
+ env:
+ MY_VAR: ${var.foo}
+ content: echo "$MY_VAR"
 
 
 Exit code: 1
 
 >>> [CLI] bundle run foo
-Error: Found ${FOO} in script one. Interpolation syntax ${...} is not allowed in scripts
+Error: Found ${FOO} in script one.content. Interpolation syntax ${...} is not supported in script content
  at scripts.one.content
  in databricks.yml:6:14
 
-We do not support the ${...} interpolation syntax in scripts because
-it's ambiguous whether it's a variable reference or reference to an
-environment variable.
+The ${...} syntax is not allowed in script content. To use a bundle value,
+declare an environment variable in the script's "env:" section and reference it
+from "content" with $NAME:
 
-Error: Found ${var.BAR} in script two. Interpolation syntax ${...} is not allowed in scripts
+ scripts:
+ one:
+ env:
+ MY_VAR: ${var.foo}
+ content: echo "$MY_VAR"
+
+Error: Found ${var.BAR} in script two.content. Interpolation syntax ${...} is not supported in script content
  at scripts.two.content
  in databricks.yml:8:14
 
-We do not support the ${...} interpolation syntax in scripts because
-it's ambiguous whether it's a variable reference or reference to an
-environment variable.
+The ${...} syntax is not allowed in script content. To use a bundle value,
+declare an environment variable in the script's "env:" section and reference it
+from "content" with $NAME:
+
+ scripts:
+ two:
+ env:
+ MY_VAR: ${var.foo}
+ content: echo "$MY_VAR"
 
 
 Exit code: 1

bundle/config/root.go

@@ -24,6 +24,13 @@ import (
 type Script struct {
 	// Content of the script to be executed.
 	Content string `json:"content"`
+
+	// Env is a map of environment variables exported when running the script.
+	// Values may reference ${bundle.*}, ${workspace.*}, ${var.*} (or
+	// ${variables.*}); other prefixes are rejected at validation time.
+	// Use this to pass bundle configuration into a script's shell environment
+	// without polluting the script content with DABs interpolation syntax.
+	Env map[string]string `json:"env,omitempty"`
 }
 
 type Root struct { //nolint:recvcheck // value receivers for read-only accessors, pointer for mutators

bundle/config/validate/scripts.go

@@ -4,12 +4,13 @@ import (
 	"context"
 	"fmt"
 	"maps"
-	"regexp"
 	"slices"
+	"strings"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/dynvar"
 )
 
 type validateScripts struct{}
@@ -22,44 +23,89 @@ func (f *validateScripts) Name() string {
 	return "validate:scripts"
 }
 
-func (f *validateScripts) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
-	diags := diag.Diagnostics{}
+// allowedEnvRefPrefixes are the variable prefixes that may appear in a
+// script's "env:" section. These match the prefixes resolved before scripts
+// execute (defaultPrefixes in resolve_variable_references.go); "var" is the
+// shorthand for "variables".
+var allowedEnvRefPrefixes = []string{"bundle", "workspace", "var", "variables"}
 
-	re := regexp.MustCompile(`\$\{.*\}`)
+func (f *validateScripts) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	var diags diag.Diagnostics
 
 	// Sort the scripts to have a deterministic order for the
 	// generated diagnostics.
 	scriptKeys := slices.Sorted(maps.Keys(b.Config.Scripts))
 
 	for _, k := range scriptKeys {
  script := b.Config.Scripts[k]
-p := dyn.NewPath(dyn.Key("scripts"), dyn.Key(k), dyn.Key("content"))
+contentPath := dyn.NewPath(dyn.Key("scripts"), dyn.Key(k), dyn.Key("content"))
 
  if script.Content == "" {
  diags = append(diags, diag.Diagnostic{
  Severity: diag.Error,
  Summary: fmt.Sprintf("Script %s has no content", k),
- Paths: []dyn.Path{p},
+ Paths: []dyn.Path{contentPath},
  })
  continue
  }
 
- v, err := dyn.GetByPath(b.Config.Value(), p)
- if err != nil {
- return diags.Extend(diag.FromErr(err))
+ diags = diags.Extend(validateScriptContent(b, k, script.Content, contentPath))
+ diags = diags.Extend(validateScriptEnv(b, k, script.Env))
+	}
+
+	return diags
+}
+
+// validateScriptContent rejects any ${...} reference in a script's content.
+// Content is passed to the shell as-is, so ${...} would be ambiguous with a
+// bundle reference; reference a declared env entry with $NAME instead.
+func validateScriptContent(b *bundle.Bundle, key, content string, p dyn.Path) diag.Diagnostics {
+	ref, ok := dynvar.NewRef(dyn.V(content))
+	if !ok {
+ return nil
+	}
+
+	first := ref.Matches[0][0]
+	v, _ := dyn.GetByPath(b.Config.Value(), p)
+	return diag.Diagnostics{{
+ Severity: diag.Error,
+ Summary: fmt.Sprintf("Found %s in script %s.content. Interpolation syntax ${...} is not supported in script content", first, key),
+ Detail: `The ${...} syntax is not allowed in script content. To use a bundle value,
+declare an environment variable in the script's "env:" section and reference it
+from "content" with $NAME:
+
+ scripts:
+ ` + key + `:
+ env:
+ MY_VAR: ${var.foo}
+ content: echo "$MY_VAR"`,
+ Locations: v.Locations(),
+ Paths: []dyn.Path{p},
+	}}
+}
+
+func validateScriptEnv(b *bundle.Bundle, key string, env map[string]string) diag.Diagnostics {
+	var diags diag.Diagnostics
+
+	for _, name := range slices.Sorted(maps.Keys(env)) {
+ ref, ok := dynvar.NewRef(dyn.V(env[name]))
+ if !ok {
+ continue
  }
 
- // Check for interpolation syntax
- match := re.FindString(script.Content)
- if match != "" {
+ envValuePath := dyn.NewPath(dyn.Key("scripts"), dyn.Key(key), dyn.Key("env"), dyn.Key(name))
+ v, _ := dyn.GetByPath(b.Config.Value(), envValuePath)
+
+ for _, refPath := range ref.References() {
+ prefix, _, _ := strings.Cut(refPath, ".")
+ if slices.Contains(allowedEnvRefPrefixes, prefix) {
+ continue
+ }
  diags = append(diags, diag.Diagnostic{
- Severity: diag.Error,
- Summary: fmt.Sprintf("Found %s in script %s. Interpolation syntax ${...} is not allowed in scripts", match, k),
- Detail: `We do not support the ${...} interpolation syntax in scripts because
-it's ambiguous whether it's a variable reference or reference to an
-environment variable.`,
+ Severity: diag.Error,
+ Summary: fmt.Sprintf("${%s} cannot be used in scripts.%s.env.%s; only ${bundle.*}, ${workspace.*}, and ${var.*} are resolved before scripts execute", refPath, key, name),
  Locations: v.Locations(),
- Paths: []dyn.Path{p},
+ Paths: []dyn.Path{envValuePath},
  })
  }
 	}

bundle/internal/schema/annotations.yml

@@ -1990,6 +1990,9 @@ scripts:
  "content":
  "description": |-
  PLACEHOLDER
+ "env":
+ "description": |-
+ A map of environment variables to export when running the script. Values may reference `${bundle.*}`, `${workspace.*}`, and `${var.*}` and are resolved before the script runs; reference them from `content` with `$NAME`.
 sync:
  "description": |-
  The files and file paths to include or exclude in the bundle.

cmd/bundle/run.go

@@ -10,6 +10,7 @@ import (
 	"slices"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/env"
 	"github.com/databricks/cli/bundle/resources"
 	"github.com/databricks/cli/bundle/run"
@@ -165,7 +166,7 @@ Example usage:
  if len(runArgs) > 0 {
  return fmt.Errorf("additional arguments are not supported for scripts. Got: %v. We recommend using environment variables to pass runtime arguments to a script. For example: FOO=bar databricks bundle run my_script", runArgs)
  }
- return executeScript(b.Config.Scripts[key].Content, cmd, b)
+ return executeScript(b.Config.Scripts[key], cmd, b)
  }
 
  return nil
@@ -276,8 +277,14 @@ func scriptEnv(cmd *cobra.Command, b *bundle.Bundle) []string {
 	return out
 }
 
-func executeScript(content string, cmd *cobra.Command, b *bundle.Bundle) error {
-	return execv.Shell(content, b.BundleRootPath, scriptEnv(cmd, b))
+func executeScript(script config.Script, cmd *cobra.Command, b *bundle.Bundle) error {
+	env := scriptEnv(cmd, b)
+	// Append after the auth/target variables so a script's env: section takes
+	// precedence on collision (os/exec uses the last value for a duplicate key).
+	for _, name := range slices.Sorted(maps.Keys(script.Env)) {
+ env = append(env, name+"="+script.Env[name])
+	}
+	return execv.Shell(script.Content, b.BundleRootPath, env)
 }
 
 func executeInline(cmd *cobra.Command, args []string, b *bundle.Bundle) error {