Add `sensitive_fields` to resources.yml by andrewnester · Pull Request #6192 · databricks/cli

acceptance/bundle/invariant/configs/secret.yml.tmpl

-Original file line number
+Diff line change
@@ -0,0 +1,18 @@
+bundle:
+ name: test-bundle-$UNIQUE_NAME
+variables:
+ secret_value:
+ description: The value of the secret
+resources:
+ secrets:
+ foo:
+ catalog_name: main
+ schema_name: default
+ name: test-secret-$UNIQUE_NAME
+ value: ${var.secret_value}
+ grants:
+ - principal: account users
+ privileges:
+ - MANAGE

acceptance/bundle/invariant/configs/secret.yml.tmpl-init.sh

Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		export BUNDLE_VAR_secret_value="secret-value"

acceptance/bundle/invariant/continue_293/test.toml

-Original file line number
+Diff line change
@@ Expand Up @@
 # The 1000-task scale case is covered by no_drift. Running it here adds ~1.5 min
 # per variant (two full deploys at 1000 tasks) without incremental coverage.
 EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"]
+# secret resource is not supported on v0.293.0
+EnvMatrixExclude.no_secret = ["INPUT_CONFIG=secret.yml.tmpl"]

acceptance/bundle/invariant/delete_idempotent/out.test.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

acceptance/bundle/invariant/destroy_idempotent/out.test.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

acceptance/bundle/invariant/destroy_idempotent/test.toml

-Original file line number
+Diff line change
@@ Expand Up / @@ -3,3 +3,8 @@ EnvMatrix.READPLAN = ["", "1"] @@
 # Snapshot of pre-destroy state used to re-run destroy on state that still
 # references the (now-gone) resources; may linger if the test fails mid-run.
 Ignore = [".databricks.backup"]
+[EnvMatrixExclude]
+# Secrets have sensitive fields (json:"-") that are stripped when a plan is serialized to JSON,
+# so deploying from a pre-computed plan file creates the secret with an empty value.
+no_secret_with_readplan = ["READPLAN=1", "INPUT_CONFIG=secret.yml.tmpl"]

acceptance/bundle/invariant/migrate/test.toml

-Original file line number
+Diff line change
@@ Expand Up @@
 # volume schema_name ("inconsistent final plan"). Covered by no_drift on direct.
 EnvMatrixExclude.no_volume_uppercase = ["INPUT_CONFIG=volume_uppercase_name.yml.tmpl"]
+EnvMatrixExclude.no_secret = ["INPUT_CONFIG=secret.yml.tmpl"]
 # Terraform types sampling_fraction as an integer and truncates 0.5; covered by no_drift.
 EnvMatrixExclude.no_model_serving_endpoint_telemetry = ["INPUT_CONFIG=model_serving_endpoint_telemetry.yml.tmpl"]

acceptance/bundle/invariant/no_drift/out.test.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

acceptance/bundle/invariant/no_drift/test.toml

-Original file line number
+Diff line change
@@ -1 +1,6 @@
 EnvMatrix.READPLAN = ["", "1"]
+[EnvMatrixExclude]
+# Secrets have sensitive fields (json:"-") that are stripped when a plan is serialized to JSON,
+# so deploying from a pre-computed plan file creates the secret with an empty value.
+no_secret_with_readplan = ["READPLAN=1", "INPUT_CONFIG=secret.yml.tmpl"]

acceptance/bundle/invariant/test.toml

-Original file line number
+Diff line change
@@ Expand Up / @@ -65,6 +65,7 @@ EnvMatrix.INPUT_CONFIG = [ @@
  "schema_empty_grants.yml.tmpl",
  "schema_grant_ref.yml.tmpl",
  "schema_uppercase_name.yml.tmpl",
+ "secret.yml.tmpl",
  "secret_scope.yml.tmpl",
  "secret_scope_default_backend_type.yml.tmpl",
  "sql_warehouse.yml.tmpl",
@@ Expand Down @@

acceptance/bundle/resources/secrets/basic/output.txt

-Original file line number
+Diff line change
@@ -1,12 +1,24 @@
->>> [CLI] bundle validate --var secret_value=initial-secret-value
-Name: test-bundle-[UNIQUE_NAME]
-Target: default
-Workspace:
- User: [USERNAME]
- Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
+>>> [CLI] bundle validate --var secret_value=initial-secret-value -o json
+>>> cat validate.json
+{
+ "secret1": {
+ "catalog_name": "main",
+ "comment": "Test secret for acceptance testing",
+ "name": "test_secret_[UNIQUE_NAME]",
+ "schema_name": "default",
+ "value": "[redacted]"
+ }
+}
-Validation OK!
+>>> cat validate.json
+{
+ "secret_value": {
+ "description": "Test secret value",
+ "value": "[redacted]"
+ }
+}
 >>> [CLI] bundle summary --var secret_value=initial-secret-value
 Name: test-bundle-[UNIQUE_NAME]
@@ Expand Down @@

acceptance/bundle/resources/secrets/basic/script

-Original file line number
+Diff line change
@@ Expand Up / @@ -6,7 +6,11 @@ cleanup() { @@
 }
 trap cleanup EXIT
-trace $CLI bundle validate --var secret_value=initial-secret-value
+trace $CLI bundle validate --var secret_value=initial-secret-value -o json > validate.json
+trace cat validate.json | jq -r ".resources.secrets"
+trace cat validate.json | jq -r ".variables"
+rm validate.json
 trace $CLI bundle summary --var secret_value=initial-secret-value
 trace $CLI bundle deploy --var secret_value=initial-secret-value
@@ Expand Down @@

acceptance/bundle/resources/secrets/update-value/output.txt

-Original file line number
+Diff line change
@@ Expand Up / @@ -12,7 +12,7 @@ @@
  "comment": "Test secret",
  "name": "test_secret",
  "schema_name": "default",
- "value": ""
+ "value": "[redacted]"
  }
  }
  }
@@ Expand DownExpand Up / @@ -40,6 +40,57 @@ Deployment complete! @@
 secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Test secret' value=''
 === Update secret value by passing a different variable value
+>>> [CLI] bundle plan --var secret_value=updated-secret-value -o json
+{
+ "plan_version": 2,
+ "cli_version": "[CLI_VERSION]",
+ "lineage": "[UUID]",
+ "serial": 1,
+ "plan": {
+ "resources.secrets.secret1": {
+ "action": "update",
+ "new_state": {
+ "value": {
+ "catalog_name": "main",
+ "comment": "Test secret",
+ "name": "test_secret",
+ "schema_name": "default",
+ "value": "[redacted]"
+ }
+ },
+ "remote_state": {
+ "catalog_name": "main",
+ "comment": "Test secret",
+ "create_time": "[TIMESTAMP]",
+ "created_by": "test-user@databricks.com",
+ "effective_owner": "test-user@databricks.com",
+ "effective_value": "[redacted]",
+ "full_name": "main.default.test_secret",
+ "metastore_id": "test-metastore-id",
+ "name": "test_secret",
+ "owner": "test-user@databricks.com",
+ "schema_name": "default",
+ "update_time": "[TIMESTAMP]",
+ "updated_by": "test-user@databricks.com",
+ "value": "[redacted]"
+ },
+ "changes": {
+ "owner": {
+ "action": "skip",
+ "reason": "spec:input_only",
+ "remote": "test-user@databricks.com"
+ },
+ "value": {
+ "action": "update",
+ "old": "",
+ "new": "[redacted]",
+ "remote": "[redacted]"
+ }
+ }
+ }
+ }
+}
 >>> [CLI] bundle deploy --var secret_value=updated-secret-value
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
 Deploying resources...
@@ Expand Down @@

acceptance/bundle/resources/secrets/update-value/script

-Original file line number
+Diff line change
@@ Expand Up / @@ -5,6 +5,7 @@ trace print_requests.py //unity-catalog @@
 read_state.py secrets secret1 catalog_name schema_name name comment value
 title "Update secret value by passing a different variable value"
+trace $CLI bundle plan --var secret_value=updated-secret-value -o json
 trace $CLI bundle deploy --var secret_value=updated-secret-value
 trace print_requests.py //unity-catalog
 read_state.py secrets secret1 catalog_name schema_name name comment value
@@ Expand Down @@

acceptance/invariant_test.go

-Original file line number
+Diff line change
@@ Expand Up / @@ -23,8 +23,6 @@ const invariantConfigsDir = "bundle/invariant/configs" @@
 // the test fails if an entry here is actually covered, so the list only shrinks.
 var LackingInvariantTest = map[string]bool{
 	"quality_monitors": true,
-	"secrets": true,
-	"secrets.grants": true,
 }
 // TestInvariantConfigsCoverage ensures that the invariant test configs in
@@ Expand Down @@

bundle/direct/apply.go

Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState,
 return err
	}

	err = db.SaveState(d.ResourceKey, newID, newState, d.DependsOn)
	err = d.saveState(db, newID, newState, d.DependsOn)
	if err != nil {
 return fmt.Errorf("saving state after creating id=%s: %w", newID, err)
	}
Expand DownExpand Up@@ -163,7 +163,7 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState,
 return fmt.Errorf("deleting state id=%s: %w", id, err)
 }
	} else {
 err = db.SaveState(d.ResourceKey, id, newState, d.DependsOn)
 err = d.saveState(db, id, newState, d.DependsOn)
 if err != nil {
 return fmt.Errorf("saving state id=%s: %w", id, err)
 }
Expand DownExpand Up@@ -208,7 +208,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment
 return err
	}

	err = db.SaveState(d.ResourceKey, newID, newState, d.DependsOn)
	err = d.saveState(db, newID, newState, d.DependsOn)
	if err != nil {
 return fmt.Errorf("saving state id=%s: %w", oldID, err)
	}
Expand DownExpand Up@@ -291,14 +291,23 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState,
 return fmt.Errorf("resizing id=%s: %w", id, err)
	}

	err = db.SaveState(d.ResourceKey, id, newState, d.DependsOn)
	err = d.saveState(db, id, newState, d.DependsOn)
	if err != nil {
 return fmt.Errorf("saving state id=%s: %w", id, err)
	}

	return nil
}

// saveState saves a state with sensitive fields replaced by a placeholder value so secrets are never written
// to disk in plaintext.
func (d *DeploymentUnit) saveState(db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry) error {
	if err := zeroSensitiveFields(d.Adapter, state); err != nil {
 return fmt.Errorf("redacting state: %w", err)
	}
	return db.SaveState(d.ResourceKey, newID, state, dependsOn)
}

func parseState(destType reflect.Type, raw json.RawMessage) (any, error) {
	destPtr := reflect.New(destType).Interface()
	err := json.Unmarshal(raw, destPtr)
Expand Down

bundle/direct/bundle_plan.go

-Original file line number
+Diff line change
@@ Expand Up @@
  }
 	}
+	for resourceKey, entry := range plan.Plan {
+ adapter, err := b.getAdapterForKey(resourceKey)
+ if err != nil {
+ return nil, fmt.Errorf("redacting plan entry %s: %w", resourceKey, err)
+ }
+ if err := redactPlanEntry(adapter, entry); err != nil {
+ return nil, fmt.Errorf("redacting plan entry %s: %w", resourceKey, err)
+ }
+	}
 	return plan, nil
 }
@@ Expand DownExpand Up @@
  return strings.Compare(a.Label, b.Label)
  })
+ // Store an unredacted copy in the cache so Apply can deploy with the
+ // actual values. The original newStateConfig is redacted below and used
+ // only for the plan output.
+ stateType := adapter.StateType()
+ cacheCopyPtr := reflect.New(stateType.Elem())
+ cacheCopyPtr.Elem().Set(reflect.ValueOf(newStateConfig).Elem())
+ b.StateCache.Store(node, &structvar.StructVar{
+ Value: cacheCopyPtr.Interface(),
+ Refs: refs,
+ })
+ // Redact sensitive fields before serialising. Sensitive values always come
+ // from bundle variables (enforced by ValidateSecretValueIsVariable), which
+ // are resolved before plan time, so SyncToJSON (called when cross-resource
+ // refs are resolved during planNode) will never re-serialise these fields.
+ if err := redactStruct(adapter, newStateConfig); err != nil {
+ return nil, fmt.Errorf("%s: cannot redact state: %w", node, err)
+ }
  newState := &structvar.StructVar{
  Value: newStateConfig,
  Refs: refs,
  }
- // Store in cache for use during planning phase
- b.StateCache.Store(node, newState)
- // Convert to JSON for serialization in plan
+ // Convert to JSON for serialization in plan (values already redacted above).
  newStateJSON, err := newState.ToJSON()
  if err != nil {
  return nil, fmt.Errorf("%s: cannot serialize state: %w", node, err)
@@ Expand Down @@

bundle/direct/dresources/adapter.go

-Original file line number
+Diff line change
@@ Expand Up / @@ -312,7 +312,8 @@ func (a *Adapter) validate() error { @@
 	// If RemapState is implemented, validate its signature.
 	// Otherwise require remote type to equal state type so remapping isn't needed.
 	if a.remapState != nil {
- validations = append(validations,
+ validations = append(
+ validations,
  "RemapState input", a.remapState.InTypes[0], remoteType,
  "RemapState return", a.remapState.OutTypes[0], stateType,
  )
@@ Expand DownExpand Up @@
 	return a.generatedResourceConfig
 }
+// GetSensitiveFields returns the list of sensitive fields for the resource.
+func (a *Adapter) GetSensitiveFields() []string {
+	var fields []string
+	for _, r := range a.resourceConfig.SensitiveFields {
+ fields = append(fields, r.Field.String())
+	}
+	return fields
+}
 // FieldTriggersRecreate reports whether a local change to the field forces a
 // delete + create. Both recreate_on_changes and provided_id_fields do this, so a
 // caller that knows the ID is preserved can conclude the field is unchanged.
@@ Expand Down @@

bundle/direct/dresources/config.go

-Original file line number
+Diff line change
@@ Expand Up / @@ -79,6 +79,9 @@ type ResourceLifecycleConfig struct { @@
 	// BackendDefaults: fields where the backend may set defaults.
 	// When old and new are nil but remote is set, and the remote value matches allowed values (if specified), the change is skipped.
 	BackendDefaults []BackendDefaultRule `yaml:"backend_defaults,omitempty"`
+	// SensitiveFields: fields that are sensitive and should not be logged.
+	SensitiveFields []FieldRule `yaml:"sensitive_fields,omitempty"`
 }
 // Config is the root configuration structure for resource lifecycle behavior.
@@ Expand All / @@ -100,6 +103,7 @@ var empty = ResourceLifecycleConfig{ @@
 	UpdatableIDFields: nil,
 	NormalizeSlash: nil,
 	BackendDefaults: nil,
+	SensitiveFields: nil,
 }
 func mustParseConfig(data []byte) func() *Config {
@@ Expand Down @@

Navigation Menu

Uh oh!

Add `sensitive_fields` to resources.yml#6192

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Add sensitive_fields to resources.yml#6192

Uh oh!

Add sensitive_fields to resources.yml #6192

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Add `sensitive_fields` to resources.yml#6192

Add `sensitive_fields` to resources.yml #6192