Make an explicitly selected profile take precedence over auth environment variables

NEXT_CHANGELOG.md

@@ -6,6 +6,8 @@
 
 ### CLI
 
+* An explicitly selected profile (`--profile` or a bundle's `workspace.profile`) now takes precedence over auth environment variables (`DATABRICKS_HOST`, `DATABRICKS_TOKEN`, etc.) instead of being silently shadowed by them; env vars still fill auth fields the profile leaves empty ([#5096](https://github.com/databricks/cli/issues/5096)).
+
 ### Bundles
 
 ### Dependency updates

acceptance/cmd/api/default-profile-vs-env/output.txt

@@ -0,0 +1,22 @@
+
+=== api without --profile uses env auth, ignoring default_profile (#5616)
+
+>>> [CLI] api get /api/2.0/clusters/list
+{}
+
+>>> print_requests.py --get //api/2.0/clusters/list
+{
+ "headers": {
+ "Authorization": [
+ "Bearer [DATABRICKS_TOKEN]"
+ ],
+ "User-Agent": [
+ "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/api_get cmd-exec-id/[UUID] interactive/none auth/pat"
+ ],
+ "X-Databricks-Workspace-Id": [
+ "[NUMID]"
+ ]
+ },
+ "method": "GET",
+ "path": "/api/2.0/clusters/list"
+}

acceptance/cmd/api/default-profile-vs-env/script

@@ -0,0 +1,19 @@
+sethome "./home"
+
+# A default profile with conflicting (basic) auth on a different host. The env
+# below still points a PAT at the test server. Without the #5616 guard, the
+# default profile would be pinned and merged with the env PAT, failing with
+# "more than one authorization method configured".
+cat > "./home/.databrickscfg" <<EOF
+[__settings__]
+default_profile = other
+
+[other]
+host = https://other.cloud.databricks.test
+username = user
+password = pass
+EOF
+
+title "api without --profile uses env auth, ignoring default_profile (#5616)\n"
+MSYS_NO_PATHCONV=1 trace $CLI api get /api/2.0/clusters/list
+trace print_requests.py --get //api/2.0/clusters/list

acceptance/cmd/api/default-profile-vs-env/test.toml

@@ -0,0 +1,3 @@
+Ignore = [
+ "home",
+]

acceptance/cmd/api/profile-overrides-env/output.txt

@@ -0,0 +1,44 @@
+
+=== api --profile overrides auth env vars (#5096)
+
+>>> [CLI] api get /api/2.0/clusters/list --profile my-workspace
+{}
+
+>>> print_requests.py --get //api/2.0/clusters/list
+{
+ "headers": {
+ "Authorization": [
+ "Bearer [DATABRICKS_TOKEN]"
+ ],
+ "User-Agent": [
+ "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/api_get cmd-exec-id/[UUID] interactive/none auth/pat"
+ ],
+ "X-Databricks-Workspace-Id": [
+ "[NUMID]"
+ ]
+ },
+ "method": "GET",
+ "path": "/api/2.0/clusters/list"
+}
+
+=== api host-only --profile fills the token from the environment (#5096)
+
+>>> [CLI] api get /api/2.0/clusters/list --profile host-only
+{}
+
+>>> print_requests.py --get //api/2.0/clusters/list
+{
+ "headers": {
+ "Authorization": [
+ "Bearer [DATABRICKS_TOKEN]"
+ ],
+ "User-Agent": [
+ "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/api_get cmd-exec-id/[UUID] interactive/none auth/pat"
+ ],
+ "X-Databricks-Workspace-Id": [
+ "[NUMID]"
+ ]
+ },
+ "method": "GET",
+ "path": "/api/2.0/clusters/list"
+}

acceptance/cmd/api/profile-overrides-env/script

@@ -0,0 +1,29 @@
+sethome "./home"
+
+# One profile with full credentials, one host-only; both point at the test
+# server while the auth env vars below point elsewhere.
+cat > "./home/.databrickscfg" <<EOF
+[my-workspace]
+host = $DATABRICKS_HOST
+token = $DATABRICKS_TOKEN
+
+[host-only]
+host = $DATABRICKS_HOST
+EOF
+
+# direnv-style auth env vars for a different workspace; before #5096 these
+# shadowed the profile selected with --profile.
+real_token=$DATABRICKS_TOKEN
+export DATABRICKS_HOST=https://dev.cloud.databricks.test
+export DATABRICKS_TOKEN=dev-token
+
+title "api --profile overrides auth env vars (#5096)\n"
+MSYS_NO_PATHCONV=1 trace $CLI api get /api/2.0/clusters/list --profile my-workspace
+trace print_requests.py --get //api/2.0/clusters/list
+
+# Host-only profile: the profile wins for the host, but env fills the token it
+# omits (#5096).
+export DATABRICKS_TOKEN=$real_token
+title "api host-only --profile fills the token from the environment (#5096)\n"
+MSYS_NO_PATHCONV=1 trace $CLI api get /api/2.0/clusters/list --profile host-only
+trace print_requests.py --get //api/2.0/clusters/list

acceptance/cmd/api/profile-overrides-env/test.toml

@@ -0,0 +1,3 @@
+Ignore = [
+ "home",
+]

acceptance/cmd/auth/describe/account-host-with-workspace-id/output.txt

@@ -7,10 +7,10 @@ Account ID: acct-123
 Authenticated with: pat
 -----
 Current configuration:
- ✓ host: [DATABRICKS_URL] (from DATABRICKS_HOST environment variable)
+ ✓ host: [DATABRICKS_URL] (from [TEST_TMP_DIR]/home/.databrickscfg config file)
  ✓ account_id: acct-123 (from [TEST_TMP_DIR]/home/.databrickscfg config file)
  ✓ workspace_id: [NUMID] (from [TEST_TMP_DIR]/home/.databrickscfg config file)
- ✓ token: ******** (from DATABRICKS_TOKEN environment variable)
+ ✓ token: ******** (from [TEST_TMP_DIR]/home/.databrickscfg config file)
  ✓ profile: acct-with-ws (from --profile flag)
  ✓ databricks_cli_path: [CLI]
  ✓ auth_type: pat

acceptance/cmd/auth/describe/profile-overrides-env/output.txt

@@ -0,0 +1,36 @@
+
+=== Describe with --profile overrides auth env vars (#5096)
+
+>>> [CLI] auth describe --profile my-workspace
+Host: [DATABRICKS_URL]
+User: [USERNAME]
+Authenticated with: pat
+-----
+Current configuration:
+ ✓ host: [DATABRICKS_URL] (from [TEST_TMP_DIR]/home/.databrickscfg config file)
+ ✓ workspace_id: [NUMID]
+ ✓ token: ******** (from [TEST_TMP_DIR]/home/.databrickscfg config file)
+ ✓ profile: my-workspace (from --profile flag)
+ ✓ databricks_cli_path: [CLI]
+ ✓ auth_type: pat
+ ✓ rate_limit: [NUMID] (from DATABRICKS_RATE_LIMIT environment variable)
+ ✓ cloud: AWS
+ ✓ discovery_url: [DATABRICKS_URL]/oidc/.well-known/oauth-authorization-server
+
+=== Describe with a host-only --profile fills the token from the environment (#5096)
+
+>>> [CLI] auth describe --profile host-only
+Host: [DATABRICKS_URL]
+User: [USERNAME]
+Authenticated with: pat
+-----
+Current configuration:
+ ✓ host: [DATABRICKS_URL] (from [TEST_TMP_DIR]/home/.databrickscfg config file)
+ ✓ workspace_id: [NUMID]
+ ✓ token: ******** (from DATABRICKS_TOKEN environment variable)
+ ✓ profile: host-only (from --profile flag)
+ ✓ databricks_cli_path: [CLI]
+ ✓ auth_type: pat
+ ✓ rate_limit: [NUMID] (from DATABRICKS_RATE_LIMIT environment variable)
+ ✓ cloud: AWS
+ ✓ discovery_url: [DATABRICKS_URL]/oidc/.well-known/oauth-authorization-server

acceptance/cmd/auth/describe/profile-overrides-env/script

@@ -0,0 +1,26 @@
+sethome "./home"
+
+# A profile carries full credentials; a second profile carries only a host.
+cat > "./home/.databrickscfg" <<EOF
+[my-workspace]
+host = $DATABRICKS_HOST
+token = $DATABRICKS_TOKEN
+
+[host-only]
+host = $DATABRICKS_HOST
+EOF
+
+# direnv-style auth env vars for a different workspace; before #5096 these
+# shadowed the profile selected with --profile.
+real_token=$DATABRICKS_TOKEN
+export DATABRICKS_HOST=https://dev.cloud.databricks.test
+export DATABRICKS_TOKEN=dev-token
+
+title "Describe with --profile overrides auth env vars (#5096)\n"
+trace $CLI auth describe --profile my-workspace
+
+# Host-only profile: the profile wins for the host, but env fills the token it
+# omits (#5096).
+export DATABRICKS_TOKEN=$real_token
+title "Describe with a host-only --profile fills the token from the environment (#5096)\n"
+trace $CLI auth describe --profile host-only

acceptance/cmd/auth/describe/profile-overrides-env/test.toml

@@ -0,0 +1,3 @@
+Ignore = [
+ "home"
+]

bundle/config/workspace.go

@@ -182,9 +182,14 @@ func (w *Workspace) Client(ctx context.Context) (*databricks.WorkspaceClient, er
 
 	cfg := w.Config(ctx)
 
-	// If only the host is configured, we try and unambiguously match it to
-	// a profile in the user's databrickscfg file. Override the default loaders.
-	if w.Host != "" && w.Profile == "" {
+	switch {
+	case w.Profile != "":
+ // An explicit profile wins over auth env vars (#5096).
+ // ValidateConfigAndProfileHost below still checks host agreement.
+ cfg.Loaders = databrickscfg.ProfileAuthLoaders
+	case w.Host != "":
+ // If only the host is configured, we try and unambiguously match it to
+ // a profile in the user's databrickscfg file. Override the default loaders.
  cfg.Loaders = []config.Loader{
  // Load auth creds from env vars
  config.ConfigAttributes,

bundle/config/workspace_test.go

@@ -155,6 +155,78 @@ func TestWorkspaceClientNormalizesHostBeforeProfileResolution(t *testing.T) {
 	assert.Equal(t, "ws2", client.Config.Profile)
 }
 
+func TestWorkspaceClientProfileOverridesAuthEnv(t *testing.T) {
+	// An explicit profile must win over auth env vars (#5096).
+	setupWorkspaceTest(t)
+
+	err := databrickscfg.SaveToProfile(t.Context(), &config.Config{
+ Profile: "tst",
+ Host: "https://tst.cloud.databricks.test",
+ Token: "tst-token",
+	})
+	require.NoError(t, err)
+
+	// direnv-style auth env vars pointing at a different (dev) workspace.
+	t.Setenv("DATABRICKS_HOST", "https://dev.cloud.databricks.test")
+	t.Setenv("DATABRICKS_TOKEN", "dev-token")
+
+	w := Workspace{Profile: "tst"}
+	client, err := w.Client(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, "tst", client.Config.Profile)
+	assert.Equal(t, "https://tst.cloud.databricks.test", client.Config.Host)
+	assert.Equal(t, "tst-token", client.Config.Token)
+}
+
+func TestWorkspaceClientProfileFillsAuthFromEnv(t *testing.T) {
+	// Host-only profile: the profile wins for the host, but env fills the token
+	// it omits (#5096).
+	setupWorkspaceTest(t)
+
+	err := databrickscfg.SaveToProfile(t.Context(), &config.Config{
+ Profile: "host-only",
+ Host: "https://tst.cloud.databricks.test",
+	})
+	require.NoError(t, err)
+
+	t.Setenv("DATABRICKS_TOKEN", "env-token")
+
+	w := Workspace{Profile: "host-only"}
+	client, err := w.Client(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, "host-only", client.Config.Profile)
+	assert.Equal(t, "https://tst.cloud.databricks.test", client.Config.Host)
+	// The token is not in the profile, so it is filled from the environment.
+	assert.Equal(t, "env-token", client.Config.Token)
+}
+
+func TestWorkspaceClientHostAndProfileOverridesAuthEnv(t *testing.T) {
+	// Bundle pins both workspace.host and workspace.profile: the profile wins
+	// for auth and the host check passes because they agree (#5096).
+	setupWorkspaceTest(t)
+
+	err := databrickscfg.SaveToProfile(t.Context(), &config.Config{
+ Profile: "tst",
+ Host: "https://tst.cloud.databricks.test",
+ Token: "tst-token",
+	})
+	require.NoError(t, err)
+
+	// direnv-style auth env vars pointing at a different (dev) workspace.
+	t.Setenv("DATABRICKS_HOST", "https://dev.cloud.databricks.test")
+	t.Setenv("DATABRICKS_TOKEN", "dev-token")
+
+	w := Workspace{
+ Host: "https://tst.cloud.databricks.test",
+ Profile: "tst",
+	}
+	client, err := w.Client(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, "tst", client.Config.Profile)
+	assert.Equal(t, "https://tst.cloud.databricks.test", client.Config.Host)
+	assert.Equal(t, "tst-token", client.Config.Token)
+}
+
 func TestWorkspaceConfigHTTPTimeout(t *testing.T) {
 	for _, tc := range []struct {
  envVal string

cmd/api/api.go

@@ -80,22 +80,27 @@ func makeCommand(method string) *cobra.Command {
 
  cfg := &config.Config{}
 
- // Resolve the profile mirroring MustWorkspaceClient precedence:
- // 1. --profile flag, 2. DATABRICKS_CONFIG_PROFILE env var (the SDK
- // also reads it, but setting cfg.Profile here keeps any error
-// messages we render referring to the same name), 3.
-// [__settings__].default_profile in the config file.
- if profileFlag := cmd.Flag("profile"); profileFlag != nil {
+ // Profile precedence, mirroring MustWorkspaceClient: --profile flag,
+ // then DATABRICKS_CONFIG_PROFILE (set here so our error messages name
+ // the same profile the SDK loads), then the default profile below.
+profileFlag := cmd.Flag("profile")
+hasProfileFlag := profileFlag != nil && profileFlag.Value.String() != ""
+ if hasProfileFlag {
  cfg.Profile = profileFlag.Value.String()
  }
  if cfg.Profile == "" {
  cfg.Profile = env.Get(cmd.Context(), "DATABRICKS_CONFIG_PROFILE")
  }
- if cfg.Profile == "" {
- cfg.Profile = databrickscfg.ResolveDefaultProfile(cmd.Context())
- }
 
- auth.NormalizeDatabricksConfigFromEnv(cmd.Context(), cfg)
+ if hasProfileFlag {
+ // An explicit --profile wins over auth env vars; its host makes env
+ // host normalization moot (#5096).
+ cfg.Loaders = databrickscfg.ProfileAuthLoaders
+ } else {
+ auth.NormalizeDatabricksConfigFromEnv(cmd.Context(), cfg)
+ }
+ // Skips the default profile when DATABRICKS_HOST is set (#5616).
+ root.ResolveDefaultProfile(cmd.Context(), cfg)
 
  api, err := client.New(cfg)
  if err != nil {