From 75a0eab10643fd29ecd4b503034c4dd242d2313d Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Thu, 24 Jul 2025 17:13:48 +0200 Subject: [PATCH 01/87] intercept auth header in the PopulateCurrentUser mutator --- .../config/mutator/populate_current_user.go | 63 ++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index 1e7fae629e8..0f39c8806b3 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -2,15 +2,24 @@ package mutator import ( "context" + "net/http" + "reflect" + "unsafe" + + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/iamutil" "github.com/databricks/cli/libs/tags" + "github.com/databricks/databricks-sdk-go/service/iam" ) -type populateCurrentUser struct{} +type populateCurrentUser struct { + lastKnownAuthorizationHeader string +} // PopulateCurrentUser sets the `current_user` property on the workspace. func PopulateCurrentUser() bundle.Mutator { @@ -27,7 +36,8 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. } w := b.WorkspaceClient() - me, err := w.CurrentUser.Me(ctx) + d := getDatabricksClient(w) + me, err := m.getCurrentUserWithAuthTracking(ctx, d) if err != nil { return diag.FromErr(err) } @@ -43,3 +53,52 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. return nil } + +// getCurrentUserWithAuthTracking makes the CurrentUser.Me method, caches the authorization header and returns result +func (m *populateCurrentUser) getCurrentUserWithAuthTracking(ctx context.Context, client *client.DatabricksClient) (*iam.User, error) { + var user iam.User + path := "/api/2.0/preview/scim/v2/Me" + + headers := make(map[string]string) + headers["Accept"] = "application/json" + + // Visitor to inspect request headers + headerInspector := func(req *http.Request) error { + for name, values := range req.Header { + if name != "Authorization" { + continue + } + for _, value := range values { + m.lastKnownAuthorizationHeader = value + } + } + return nil + } + + err := client.Do(ctx, http.MethodGet, path, headers, nil, nil, &user, headerInspector) + return &user, err +} + +// TODO: find a way to get the client without using reflection +func getDatabricksClient(w *databricks.WorkspaceClient) *client.DatabricksClient { + v := reflect.ValueOf(w.CurrentUser) + // value is a pointer. Keep dereferencing it until we get to the actual value + for v.Kind() == reflect.Ptr { + if v.IsNil() { + panic("nil pointer encountered") + } + v = v.Elem() + } + + clientField := v.FieldByName("client") + clientInterface := getUnexportedField(clientField) + client, ok := clientInterface.(*client.DatabricksClient) + if !ok { + panic("client is not a client.DatabricksClient") + } + return client +} + +func getUnexportedField(field reflect.Value) any { + return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface() +} From e5c1b467e8f9f31a2ecd8d11cf8e7e6ebf0c2fab Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 28 Jul 2025 16:02:28 +0200 Subject: [PATCH 02/87] cache the auth header in local dir --- bundle/bundle.go | 39 +++++- bundle/cache.go | 132 ++++++++++++++++++ .../config/mutator/populate_current_user.go | 56 ++++++++ 3 files changed, 226 insertions(+), 1 deletion(-) create mode 100644 bundle/cache.go diff --git a/bundle/bundle.go b/bundle/bundle.go index e34012580c7..b43e233a321 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -32,7 +32,10 @@ import ( "github.com/hashicorp/terraform-exec/tfexec" ) -const internalFolder = ".internal" +const ( + internalFolder = ".internal" + cacheFolder = ".cache" +) // Filename where resources are stored for DATABRICKS_BUNDLE_ENGINE=direct const resourcesFilename = "resources.json" @@ -287,6 +290,40 @@ func (b *Bundle) InternalDir(ctx context.Context) (string, error) { return dir, nil } +// BundleLevelCacheDir is used to cache components needed for the bundle that are target-independent +func (b *Bundle) BundleLevelCacheDir(ctx context.Context, cacheComponentName string) (string, error) { + if b.Config.Bundle.Target == "" { + panic("target not set") + } + + cacheDirName, exists := env.TempDir(ctx) + if !exists || cacheDirName == "" { + cacheDirName = filepath.Join( + // Anchor at bundle root directory. + b.BundleRootPath, + // Static cache directory. + ".databricks", + ) + } + + // Fixed components of the result path. + parts := []string{ + cacheDirName, + cacheFolder, + cacheComponentName, + } + + // Make directory if it doesn't exist yet. + dir := filepath.Join(parts...) + err := os.MkdirAll(dir, 0o700) + if err != nil { + return "", err + } + + libsync.WriteGitIgnore(ctx, b.BundleRootPath) + return dir, nil +} + // GetSyncIncludePatterns returns a list of user defined includes // And also adds InternalDir folder to include list for sync command // so this folder is always synced diff --git a/bundle/cache.go b/bundle/cache.go new file mode 100644 index 00000000000..2b8549ae673 --- /dev/null +++ b/bundle/cache.go @@ -0,0 +1,132 @@ +package bundle + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" +) + +// Cache provides an abstract interface for caching content to local disk. +// Implementations should handle storing and retrieving cached components +// using fingerprints for cache invalidation. +type Cache interface { + // Read retrieves cached content for the given fingerprint. + // Returns the cached data and true if found, or nil and false if not found or expired. + Read(ctx context.Context, fingerprint string) ([]byte, bool) + + // Store saves content to the cache with the given fingerprint. + // Returns an error if the cache operation fails. + Store(ctx context.Context, fingerprint string, content []byte) error + + // Clear removes all cached content from the cache directory. + Clear(ctx context.Context) error + + // ClearFingerprint removes cached content for a specific fingerprint. + ClearFingerprint(ctx context.Context, fingerprint string) error +} + +// FileCache implements the Cache interface using the local filesystem. +type FileCache struct { + cachePath string +} + +// NewFileCache creates a new filesystem-based cache at the specified path. +func NewFileCache(cachePath string) *FileCache { + return &FileCache{ + cachePath: cachePath, + } +} + +// Read retrieves cached content for the given fingerprint. +func (fc *FileCache) Read(ctx context.Context, fingerprint string) ([]byte, bool) { + if err := fc.ensureCacheDir(); err != nil { + return nil, false + } + + filePath := fc.getFilePath(fingerprint) + data, err := os.ReadFile(filePath) + if err != nil { + return nil, false + } + + return data, true +} + +// Store saves content to the cache with the given fingerprint. +func (fc *FileCache) Store(ctx context.Context, fingerprint string, content []byte) error { + if err := fc.ensureCacheDir(); err != nil { + return fmt.Errorf("failed to create cache directory: %w", err) + } + + filePath := fc.getFilePath(fingerprint) + if err := os.WriteFile(filePath, content, 0o600); err != nil { + return fmt.Errorf("failed to write cache file: %w", err) + } + + return nil +} + +// Clear removes all cached content from the cache directory. +func (fc *FileCache) Clear(ctx context.Context) error { + if _, err := os.Stat(fc.cachePath); os.IsNotExist(err) { + return nil + } + + return os.RemoveAll(fc.cachePath) +} + +// ClearFingerprint removes cached content for a specific fingerprint. +func (fc *FileCache) ClearFingerprint(ctx context.Context, fingerprint string) error { + filePath := fc.getFilePath(fingerprint) + if err := os.Remove(filePath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove cache file: %w", err) + } + return nil +} + +// ensureCacheDir creates the cache directory if it doesn't exist. +func (fc *FileCache) ensureCacheDir() error { + return os.MkdirAll(fc.cachePath, 0o700) +} + +// getFilePath returns the full file path for a given fingerprint. +func (fc *FileCache) getFilePath(fingerprint string) string { + return filepath.Join(fc.cachePath, fingerprint+".cache") +} + +// GenerateFingerprint creates a SHA256 fingerprint from the provided data. +// This is a utility function for creating consistent fingerprints. +func GenerateFingerprint(data ...any) (string, error) { + hasher := sha256.New() + + for _, item := range data { + var bytes []byte + var err error + + switch v := item.(type) { + case string: + bytes = []byte(v) + case []byte: + bytes = v + case io.Reader: + bytes, err = io.ReadAll(v) + if err != nil { + return "", fmt.Errorf("failed to read data for fingerprint: %w", err) + } + default: + bytes, err = json.Marshal(v) + if err != nil { + return "", fmt.Errorf("failed to marshal data for fingerprint: %w", err) + } + } + + hasher.Write(bytes) + } + + return hex.EncodeToString(hasher.Sum(nil)), nil +} diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index 0f39c8806b3..eb90788ccf4 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -2,6 +2,7 @@ package mutator import ( "context" + "fmt" "net/http" "reflect" "unsafe" @@ -19,6 +20,7 @@ import ( type populateCurrentUser struct { lastKnownAuthorizationHeader string + cache bundle.Cache } // PopulateCurrentUser sets the `current_user` property on the workspace. @@ -26,6 +28,43 @@ func PopulateCurrentUser() bundle.Mutator { return &populateCurrentUser{} } +// initializeCache sets up the cache for authorization headers if not already initialized +func (m *populateCurrentUser) initializeCache(ctx context.Context, b *bundle.Bundle) error { + if m.cache != nil { + return nil + } + + cacheDir, err := b.BundleLevelCacheDir(ctx, "auth") + if err != nil { + return err + } + + m.cache = bundle.NewFileCache(cacheDir) + + fmt.Printf("New cache dir initialized: %s\n", cacheDir) + + return nil +} + +// getCachedAuthorizationHeader retrieves the cached authorization header for a given host +func (m *populateCurrentUser) getCachedAuthorizationHeader(ctx context.Context, host string) (string, bool) { + if m.cache == nil { + return "", false + } + + fingerprint, err := bundle.GenerateFingerprint("auth_header", host) + if err != nil { + return "", false + } + + data, found := m.cache.Read(ctx, fingerprint) + if !found { + return "", false + } + + return string(data), true +} + func (m *populateCurrentUser) Name() string { return "PopulateCurrentUser" } @@ -35,8 +74,16 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. return nil } + // Initialize cache for authorization headers + if err := m.initializeCache(ctx, b); err != nil { + return diag.FromErr(err) + } + w := b.WorkspaceClient() d := getDatabricksClient(w) + + fmt.Printf("populateCurrentUser - getting user auth") + me, err := m.getCurrentUserWithAuthTracking(ctx, d) if err != nil { return diag.FromErr(err) @@ -70,6 +117,15 @@ func (m *populateCurrentUser) getCurrentUserWithAuthTracking(ctx context.Context } for _, value := range values { m.lastKnownAuthorizationHeader = value + // Store authorization header in cache + fmt.Printf("got lastKnownAuthorizationHeader: %s\n", value) + if m.cache != nil { + fingerprint, err := bundle.GenerateFingerprint("auth_header", req.URL.Host) + if err == nil { + fmt.Printf("storing cached auth header: %s\n", fingerprint) + _ = m.cache.Store(ctx, fingerprint, []byte(value)) + } + } } } return nil From 09d972ade821cb3724dd793c14a8c99dc873d72b Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 28 Jul 2025 17:01:18 +0200 Subject: [PATCH 03/87] store and read current user info from local cache --- .../config/mutator/populate_current_user.go | 75 ++++++++++++------- 1 file changed, 47 insertions(+), 28 deletions(-) diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index eb90788ccf4..8c7effb76a0 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -2,6 +2,8 @@ package mutator import ( "context" + "encoding/json" + "errors" "fmt" "net/http" "reflect" @@ -18,6 +20,15 @@ import ( "github.com/databricks/databricks-sdk-go/service/iam" ) +// cacheHitError is returned when a cached user is found to skip HTTP request +type cacheHitError struct { + user *iam.User +} + +func (e *cacheHitError) Error() string { + return "user found in cache" +} + type populateCurrentUser struct { lastKnownAuthorizationHeader string cache bundle.Cache @@ -46,25 +57,6 @@ func (m *populateCurrentUser) initializeCache(ctx context.Context, b *bundle.Bun return nil } -// getCachedAuthorizationHeader retrieves the cached authorization header for a given host -func (m *populateCurrentUser) getCachedAuthorizationHeader(ctx context.Context, host string) (string, bool) { - if m.cache == nil { - return "", false - } - - fingerprint, err := bundle.GenerateFingerprint("auth_header", host) - if err != nil { - return "", false - } - - data, found := m.cache.Read(ctx, fingerprint) - if !found { - return "", false - } - - return string(data), true -} - func (m *populateCurrentUser) Name() string { return "PopulateCurrentUser" } @@ -82,8 +74,6 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. w := b.WorkspaceClient() d := getDatabricksClient(w) - fmt.Printf("populateCurrentUser - getting user auth") - me, err := m.getCurrentUserWithAuthTracking(ctx, d) if err != nil { return diag.FromErr(err) @@ -116,22 +106,51 @@ func (m *populateCurrentUser) getCurrentUserWithAuthTracking(ctx context.Context continue } for _, value := range values { - m.lastKnownAuthorizationHeader = value - // Store authorization header in cache - fmt.Printf("got lastKnownAuthorizationHeader: %s\n", value) if m.cache != nil { - fingerprint, err := bundle.GenerateFingerprint("auth_header", req.URL.Host) - if err == nil { - fmt.Printf("storing cached auth header: %s\n", fingerprint) - _ = m.cache.Store(ctx, fingerprint, []byte(value)) + fingerprint, err := bundle.GenerateFingerprint("auth_header", value) + if err != nil { + panic(err) + } + cachedUserBytes, isCacheHit := m.cache.Read(ctx, fingerprint) + if isCacheHit { + var cachedUser iam.User + if err := json.Unmarshal(cachedUserBytes, &cachedUser); err == nil { + return &cacheHitError{user: &cachedUser} + } } } + m.lastKnownAuthorizationHeader = value } } return nil } err := client.Do(ctx, http.MethodGet, path, headers, nil, nil, &user, headerInspector) + + // Check if we got a cache hit error + var cacheHit *cacheHitError + if err != nil && errors.As(err, &cacheHit) { + return cacheHit.user, nil + } + + // Store authorization header in cache + if m.cache != nil && m.lastKnownAuthorizationHeader != "" { + fingerprint, err := bundle.GenerateFingerprint("auth_header", m.lastKnownAuthorizationHeader) + if err != nil { + panic(err) + } + + userBytes, err := json.Marshal(&user) + if err != nil { + return nil, err + } + + err = m.cache.Store(ctx, fingerprint, userBytes) + if err != nil { + fmt.Printf("cache store error: %s\n", err) + } + } + return &user, err } From 71de033b1fa948ffc993ad96c5dc861817669321 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 28 Jul 2025 17:15:05 +0200 Subject: [PATCH 04/87] remove unnecessary check --- bundle/bundle.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/bundle/bundle.go b/bundle/bundle.go index b43e233a321..a2f2a03f86c 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -292,10 +292,6 @@ func (b *Bundle) InternalDir(ctx context.Context) (string, error) { // BundleLevelCacheDir is used to cache components needed for the bundle that are target-independent func (b *Bundle) BundleLevelCacheDir(ctx context.Context, cacheComponentName string) (string, error) { - if b.Config.Bundle.Target == "" { - panic("target not set") - } - cacheDirName, exists := env.TempDir(ctx) if !exists || cacheDirName == "" { cacheDirName = filepath.Join( From c982a6eedcc62d1b9ae9ee7dd539abcedcc425de Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 28 Jul 2025 17:30:25 +0200 Subject: [PATCH 05/87] remove ensureCacheDir method --- bundle/cache.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/bundle/cache.go b/bundle/cache.go index 2b8549ae673..4eaeb2ff12f 100644 --- a/bundle/cache.go +++ b/bundle/cache.go @@ -44,10 +44,6 @@ func NewFileCache(cachePath string) *FileCache { // Read retrieves cached content for the given fingerprint. func (fc *FileCache) Read(ctx context.Context, fingerprint string) ([]byte, bool) { - if err := fc.ensureCacheDir(); err != nil { - return nil, false - } - filePath := fc.getFilePath(fingerprint) data, err := os.ReadFile(filePath) if err != nil { @@ -59,10 +55,6 @@ func (fc *FileCache) Read(ctx context.Context, fingerprint string) ([]byte, bool // Store saves content to the cache with the given fingerprint. func (fc *FileCache) Store(ctx context.Context, fingerprint string, content []byte) error { - if err := fc.ensureCacheDir(); err != nil { - return fmt.Errorf("failed to create cache directory: %w", err) - } - filePath := fc.getFilePath(fingerprint) if err := os.WriteFile(filePath, content, 0o600); err != nil { return fmt.Errorf("failed to write cache file: %w", err) @@ -89,11 +81,6 @@ func (fc *FileCache) ClearFingerprint(ctx context.Context, fingerprint string) e return nil } -// ensureCacheDir creates the cache directory if it doesn't exist. -func (fc *FileCache) ensureCacheDir() error { - return os.MkdirAll(fc.cachePath, 0o700) -} - // getFilePath returns the full file path for a given fingerprint. func (fc *FileCache) getFilePath(fingerprint string) string { return filepath.Join(fc.cachePath, fingerprint+".cache") From ad7f41d10faed20aedd2da89a414bd354b8db679 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 8 Sep 2025 15:08:44 +0200 Subject: [PATCH 06/87] use bearer token for fingerprint --- .../config/mutator/populate_current_user.go | 236 ++++++++++-------- 1 file changed, 135 insertions(+), 101 deletions(-) diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index 8c7effb76a0..482346815bd 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -3,15 +3,7 @@ package mutator import ( "context" "encoding/json" - "errors" "fmt" - "net/http" - "reflect" - "unsafe" - - "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/client" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" @@ -52,7 +44,7 @@ func (m *populateCurrentUser) initializeCache(ctx context.Context, b *bundle.Bun m.cache = bundle.NewFileCache(cacheDir) - fmt.Printf("New cache dir initialized: %s\n", cacheDir) + fmt.Printf("[DEBUG antonnek] New cache dir initialized: %s\n", cacheDir) return nil } @@ -66,114 +58,156 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. return nil } - // Initialize cache for authorization headers - if err := m.initializeCache(ctx, b); err != nil { - return diag.FromErr(err) - } - - w := b.WorkspaceClient() - d := getDatabricksClient(w) - - me, err := m.getCurrentUserWithAuthTracking(ctx, d) + err := m.initializeCache(ctx, b) if err != nil { - return diag.FromErr(err) - } - - b.Config.Workspace.CurrentUser = &config.User{ - ShortName: iamutil.GetShortUserName(me), - DomainFriendlyName: iamutil.GetShortUserDomainFriendlyName(me), - User: me, + fmt.Printf("[DEBUG antonnek] failed to initialize cache: %v \n", err) } + w := b.WorkspaceClient() - // Configure tagging object now that we know we have a valid client. - b.Tagging = tags.ForCloud(w.Config) - - return nil -} - -// getCurrentUserWithAuthTracking makes the CurrentUser.Me method, caches the authorization header and returns result -func (m *populateCurrentUser) getCurrentUserWithAuthTracking(ctx context.Context, client *client.DatabricksClient) (*iam.User, error) { - var user iam.User - path := "/api/2.0/preview/scim/v2/Me" - - headers := make(map[string]string) - headers["Accept"] = "application/json" - - // Visitor to inspect request headers - headerInspector := func(req *http.Request) error { - for name, values := range req.Header { - if name != "Authorization" { - continue - } - for _, value := range values { - if m.cache != nil { - fingerprint, err := bundle.GenerateFingerprint("auth_header", value) - if err != nil { - panic(err) - } - cachedUserBytes, isCacheHit := m.cache.Read(ctx, fingerprint) - if isCacheHit { - var cachedUser iam.User - if err := json.Unmarshal(cachedUserBytes, &cachedUser); err == nil { - return &cacheHitError{user: &cachedUser} - } - } - } - m.lastKnownAuthorizationHeader = value - } + // use bearer token + bearerToken := "" + tokenSource := w.Config.GetTokenSource() + if tokenSource == nil { + fmt.Printf("[DEBUG antonnek] token source not found\n") + } else { + token, err := tokenSource.Token(context.Background()) + if err != nil { + fmt.Printf("[DEBUG antonnek] error reading token source: %v \n", err) } - return nil - } - - err := client.Do(ctx, http.MethodGet, path, headers, nil, nil, &user, headerInspector) - - // Check if we got a cache hit error - var cacheHit *cacheHitError - if err != nil && errors.As(err, &cacheHit) { - return cacheHit.user, nil + bearerToken = token.AccessToken } - // Store authorization header in cache - if m.cache != nil && m.lastKnownAuthorizationHeader != "" { - fingerprint, err := bundle.GenerateFingerprint("auth_header", m.lastKnownAuthorizationHeader) + var me *iam.User + if bearerToken != "" { + fmt.Printf("[DEBUG antonnek] bearer token found: will use that for cache fingerprint\n") + fmt.Printf("[DEBUG antonnek] bearer token: %s\n", bearerToken) + fingerprint, err := bundle.GenerateFingerprint("auth_header", bearerToken) if err != nil { panic(err) } - - userBytes, err := json.Marshal(&user) - if err != nil { - return nil, err + cachedUserBytes, isCacheHit := m.cache.Read(ctx, fingerprint) + if isCacheHit { + if err := json.Unmarshal(cachedUserBytes, &me); err == nil { + fmt.Printf("[DEBUG antonnek] bearer token found: will use that for cache fingerprint\n") + } } + } - err = m.cache.Store(ctx, fingerprint, userBytes) + if me == nil { + currentUser, err := w.CurrentUser.Me(ctx) if err != nil { - fmt.Printf("cache store error: %s\n", err) + return diag.FromErr(err) } - } - - return &user, err -} - -// TODO: find a way to get the client without using reflection -func getDatabricksClient(w *databricks.WorkspaceClient) *client.DatabricksClient { - v := reflect.ValueOf(w.CurrentUser) - // value is a pointer. Keep dereferencing it until we get to the actual value - for v.Kind() == reflect.Ptr { - if v.IsNil() { - panic("nil pointer encountered") + me = currentUser + if bearerToken != "" { + userBytes, err := json.Marshal(currentUser) + if err != nil { + fmt.Printf("[DEBUG antonnek] could not serialize current user information: %v\n", err) + } + err = m.cache.Store(ctx, bearerToken, userBytes) + if err != nil { + fmt.Printf("[DEBUG antonnek] could not store user information: %v\n", err) + } else { + fmt.Printf("[DEBUG antonnek] stored user information in cache!") + } } - v = v.Elem() } - clientField := v.FieldByName("client") - clientInterface := getUnexportedField(clientField) - client, ok := clientInterface.(*client.DatabricksClient) - if !ok { - panic("client is not a client.DatabricksClient") + b.Config.Workspace.CurrentUser = &config.User{ + ShortName: iamutil.GetShortUserName(me), + DomainFriendlyName: iamutil.GetShortUserDomainFriendlyName(me), + User: me, } - return client -} -func getUnexportedField(field reflect.Value) any { - return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface() + // Configure tagging object now that we know we have a valid client. + b.Tagging = tags.ForCloud(w.Config) + + return nil } + +// getCurrentUserWithAuthTracking calls the CurrentUser.Me method, caches the authorization header and returns result +//func (m *populateCurrentUser) getCurrentUserWithAuthTracking(ctx context.Context, client *client.DatabricksClient) (*iam.User, error) { +// var user iam.User +// path := "/api/2.0/preview/scim/v2/Me" +// +// headers := make(map[string]string) +// headers["Accept"] = "application/json" +// +// // Visitor to inspect request headers +// //headerInspector := func(req *http.Request) error { +// // for name, values := range req.Header { +// // if name != "Authorization" { +// // continue +// // } +// // for _, value := range values { +// // if m.cache != nil { +// // fingerprint, err := bundle.GenerateFingerprint("auth_header", value) +// // if err != nil { +// // panic(err) +// // } +// // cachedUserBytes, isCacheHit := m.cache.Read(ctx, fingerprint) +// // if isCacheHit { +// // var cachedUser iam.User +// // if err := json.Unmarshal(cachedUserBytes, &cachedUser); err == nil { +// // return &cacheHitError{user: &cachedUser} +// // } +// // } +// // } +// // m.lastKnownAuthorizationHeader = value +// // } +// // } +// // return nil +// //} +// +// err := client.Do(ctx, http.MethodGet, path, headers, nil, nil, &user) +// +// // Check if we got a cache hit error +// var cacheHit *cacheHitError +// if err != nil && errors.As(err, &cacheHit) { +// return cacheHit.user, nil +// } +// +// // Store authorization header in cache +// if m.cache != nil && m.lastKnownAuthorizationHeader != "" { +// fingerprint, err := bundle.GenerateFingerprint("auth_header", m.lastKnownAuthorizationHeader) +// if err != nil { +// panic(err) +// } +// +// userBytes, err := json.Marshal(&user) +// if err != nil { +// return nil, err +// } +// +// err = m.cache.Store(ctx, fingerprint, userBytes) +// if err != nil { +// fmt.Printf("cache store error: %s\n", err) +// } +// } +// +// return &user, err +//} + +//// TODO: find a way to get the client without using reflection +//func getDatabricksClient(w *databricks.WorkspaceClient) *client.DatabricksClient { +// v := reflect.ValueOf(w.CurrentUser) +// // value is a pointer. Keep dereferencing it until we get to the actual value +// for v.Kind() == reflect.Ptr { +// if v.IsNil() { +// panic("nil pointer encountered") +// } +// v = v.Elem() +// } +// +// clientField := v.FieldByName("client") +// clientInterface := getUnexportedField(clientField) +// client, ok := clientInterface.(*client.DatabricksClient) +// if !ok { +// panic("client is not a client.DatabricksClient") +// } +// return client +//} + +//func getUnexportedField(field reflect.Value) any { +// return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface() +//} From 65655581dc88a51c3d3d025253fda0a50900951c Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 8 Sep 2025 17:01:10 +0200 Subject: [PATCH 07/87] fix the fingerprint when using the --- bundle/cache.go | 3 ++- bundle/config/mutator/populate_current_user.go | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bundle/cache.go b/bundle/cache.go index 4eaeb2ff12f..1f64298b8d9 100644 --- a/bundle/cache.go +++ b/bundle/cache.go @@ -115,5 +115,6 @@ func GenerateFingerprint(data ...any) (string, error) { hasher.Write(bytes) } - return hex.EncodeToString(hasher.Sum(nil)), nil + hash := hasher.Sum(nil) + return hex.EncodeToString(hash[:16]), nil } diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index 482346815bd..8de01c1ba39 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -88,7 +88,7 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. cachedUserBytes, isCacheHit := m.cache.Read(ctx, fingerprint) if isCacheHit { if err := json.Unmarshal(cachedUserBytes, &me); err == nil { - fmt.Printf("[DEBUG antonnek] bearer token found: will use that for cache fingerprint\n") + fmt.Printf("[DEBUG antonnek] user info read from cache: %s\n", fingerprint) } } } @@ -104,11 +104,12 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. if err != nil { fmt.Printf("[DEBUG antonnek] could not serialize current user information: %v\n", err) } - err = m.cache.Store(ctx, bearerToken, userBytes) + fingerprint, err := bundle.GenerateFingerprint("auth_header", bearerToken) + err = m.cache.Store(ctx, fingerprint, userBytes) if err != nil { fmt.Printf("[DEBUG antonnek] could not store user information: %v\n", err) } else { - fmt.Printf("[DEBUG antonnek] stored user information in cache!") + fmt.Printf("[DEBUG antonnek] stored user information in cache: %s\n", fingerprint) } } } From 489cb3e92246ac10bfce08761903c43e96eeeb8c Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 8 Sep 2025 17:26:27 +0200 Subject: [PATCH 08/87] refactor: extract caching logic from Apply --- .../config/mutator/populate_current_user.go | 113 +++++++++++------- 1 file changed, 72 insertions(+), 41 deletions(-) diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index 8de01c1ba39..4250996c8ee 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/iamutil" "github.com/databricks/cli/libs/tags" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/iam" ) @@ -64,34 +65,8 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. } w := b.WorkspaceClient() - // use bearer token - bearerToken := "" - tokenSource := w.Config.GetTokenSource() - if tokenSource == nil { - fmt.Printf("[DEBUG antonnek] token source not found\n") - } else { - token, err := tokenSource.Token(context.Background()) - if err != nil { - fmt.Printf("[DEBUG antonnek] error reading token source: %v \n", err) - } - bearerToken = token.AccessToken - } - - var me *iam.User - if bearerToken != "" { - fmt.Printf("[DEBUG antonnek] bearer token found: will use that for cache fingerprint\n") - fmt.Printf("[DEBUG antonnek] bearer token: %s\n", bearerToken) - fingerprint, err := bundle.GenerateFingerprint("auth_header", bearerToken) - if err != nil { - panic(err) - } - cachedUserBytes, isCacheHit := m.cache.Read(ctx, fingerprint) - if isCacheHit { - if err := json.Unmarshal(cachedUserBytes, &me); err == nil { - fmt.Printf("[DEBUG antonnek] user info read from cache: %s\n", fingerprint) - } - } - } + bearerToken := m.getBearerToken(w) + me := m.getUserFromCache(ctx, bearerToken) if me == nil { currentUser, err := w.CurrentUser.Me(ctx) @@ -99,19 +74,7 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. return diag.FromErr(err) } me = currentUser - if bearerToken != "" { - userBytes, err := json.Marshal(currentUser) - if err != nil { - fmt.Printf("[DEBUG antonnek] could not serialize current user information: %v\n", err) - } - fingerprint, err := bundle.GenerateFingerprint("auth_header", bearerToken) - err = m.cache.Store(ctx, fingerprint, userBytes) - if err != nil { - fmt.Printf("[DEBUG antonnek] could not store user information: %v\n", err) - } else { - fmt.Printf("[DEBUG antonnek] stored user information in cache: %s\n", fingerprint) - } - } + m.storeUserInCache(ctx, bearerToken, currentUser) } b.Config.Workspace.CurrentUser = &config.User{ @@ -126,6 +89,74 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. return nil } +// getBearerToken extracts the bearer token from the workspace client's token source +func (m *populateCurrentUser) getBearerToken(w *databricks.WorkspaceClient) string { + bearerToken := "" + tokenSource := w.Config.GetTokenSource() + if tokenSource == nil { + fmt.Printf("[DEBUG antonnek] token source not found\n") + } else { + token, err := tokenSource.Token(context.Background()) + if err != nil { + fmt.Printf("[DEBUG antonnek] error reading token source: %v \n", err) + } else { + bearerToken = token.AccessToken + } + } + return bearerToken +} + +// getUserFromCache attempts to retrieve user information from cache using the bearer token +func (m *populateCurrentUser) getUserFromCache(ctx context.Context, bearerToken string) *iam.User { + if bearerToken == "" || m.cache == nil { + return nil + } + + fmt.Printf("[DEBUG antonnek] bearer token found: will use that for cache fingerprint\n") + fmt.Printf("[DEBUG antonnek] bearer token: %s\n", bearerToken) + + fingerprint, err := bundle.GenerateFingerprint("auth_header", bearerToken) + if err != nil { + panic(err) + } + + cachedUserBytes, isCacheHit := m.cache.Read(ctx, fingerprint) + if isCacheHit { + var me *iam.User + if err := json.Unmarshal(cachedUserBytes, &me); err == nil { + fmt.Printf("[DEBUG antonnek] user info read from cache: %s\n", fingerprint) + return me + } + } + + return nil +} + +// storeUserInCache stores user information in cache using the bearer token as key +func (m *populateCurrentUser) storeUserInCache(ctx context.Context, bearerToken string, user *iam.User) { + if bearerToken == "" || m.cache == nil { + return + } + + userBytes, err := json.Marshal(user) + if err != nil { + fmt.Printf("[DEBUG antonnek] could not serialize current user information: %v\n", err) + return + } + + fingerprint, err := bundle.GenerateFingerprint("auth_header", bearerToken) + if err != nil { + panic(err) + } + + err = m.cache.Store(ctx, fingerprint, userBytes) + if err != nil { + fmt.Printf("[DEBUG antonnek] could not store user information: %v\n", err) + } else { + fmt.Printf("[DEBUG antonnek] stored user information in cache: %s\n", fingerprint) + } +} + // getCurrentUserWithAuthTracking calls the CurrentUser.Me method, caches the authorization header and returns result //func (m *populateCurrentUser) getCurrentUserWithAuthTracking(ctx context.Context, client *client.DatabricksClient) (*iam.User, error) { // var user iam.User From ea54f71189cd27974956e204610ba1e6a3e53cc8 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 8 Sep 2025 17:27:03 +0200 Subject: [PATCH 09/87] cleanup: remove poc code --- .../config/mutator/populate_current_user.go | 96 ------------------- 1 file changed, 96 deletions(-) diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index 4250996c8ee..2872148fd10 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -13,15 +13,6 @@ import ( "github.com/databricks/databricks-sdk-go/service/iam" ) -// cacheHitError is returned when a cached user is found to skip HTTP request -type cacheHitError struct { - user *iam.User -} - -func (e *cacheHitError) Error() string { - return "user found in cache" -} - type populateCurrentUser struct { lastKnownAuthorizationHeader string cache bundle.Cache @@ -156,90 +147,3 @@ func (m *populateCurrentUser) storeUserInCache(ctx context.Context, bearerToken fmt.Printf("[DEBUG antonnek] stored user information in cache: %s\n", fingerprint) } } - -// getCurrentUserWithAuthTracking calls the CurrentUser.Me method, caches the authorization header and returns result -//func (m *populateCurrentUser) getCurrentUserWithAuthTracking(ctx context.Context, client *client.DatabricksClient) (*iam.User, error) { -// var user iam.User -// path := "/api/2.0/preview/scim/v2/Me" -// -// headers := make(map[string]string) -// headers["Accept"] = "application/json" -// -// // Visitor to inspect request headers -// //headerInspector := func(req *http.Request) error { -// // for name, values := range req.Header { -// // if name != "Authorization" { -// // continue -// // } -// // for _, value := range values { -// // if m.cache != nil { -// // fingerprint, err := bundle.GenerateFingerprint("auth_header", value) -// // if err != nil { -// // panic(err) -// // } -// // cachedUserBytes, isCacheHit := m.cache.Read(ctx, fingerprint) -// // if isCacheHit { -// // var cachedUser iam.User -// // if err := json.Unmarshal(cachedUserBytes, &cachedUser); err == nil { -// // return &cacheHitError{user: &cachedUser} -// // } -// // } -// // } -// // m.lastKnownAuthorizationHeader = value -// // } -// // } -// // return nil -// //} -// -// err := client.Do(ctx, http.MethodGet, path, headers, nil, nil, &user) -// -// // Check if we got a cache hit error -// var cacheHit *cacheHitError -// if err != nil && errors.As(err, &cacheHit) { -// return cacheHit.user, nil -// } -// -// // Store authorization header in cache -// if m.cache != nil && m.lastKnownAuthorizationHeader != "" { -// fingerprint, err := bundle.GenerateFingerprint("auth_header", m.lastKnownAuthorizationHeader) -// if err != nil { -// panic(err) -// } -// -// userBytes, err := json.Marshal(&user) -// if err != nil { -// return nil, err -// } -// -// err = m.cache.Store(ctx, fingerprint, userBytes) -// if err != nil { -// fmt.Printf("cache store error: %s\n", err) -// } -// } -// -// return &user, err -//} - -//// TODO: find a way to get the client without using reflection -//func getDatabricksClient(w *databricks.WorkspaceClient) *client.DatabricksClient { -// v := reflect.ValueOf(w.CurrentUser) -// // value is a pointer. Keep dereferencing it until we get to the actual value -// for v.Kind() == reflect.Ptr { -// if v.IsNil() { -// panic("nil pointer encountered") -// } -// v = v.Elem() -// } -// -// clientField := v.FieldByName("client") -// clientInterface := getUnexportedField(clientField) -// client, ok := clientInterface.(*client.DatabricksClient) -// if !ok { -// panic("client is not a client.DatabricksClient") -// } -// return client -//} - -//func getUnexportedField(field reflect.Value) any { -// return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface() -//} From bdae5a501c8c0f16e859ebb2565e1122629a61e8 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 8 Sep 2025 17:33:25 +0200 Subject: [PATCH 10/87] fix lint --- bundle/config/mutator/populate_current_user.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index 2872148fd10..41bf6ad1fbe 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" @@ -14,8 +15,7 @@ import ( ) type populateCurrentUser struct { - lastKnownAuthorizationHeader string - cache bundle.Cache + cache bundle.Cache } // PopulateCurrentUser sets the `current_user` property on the workspace. From dd6943d692961360dbfc951a0eab853dc4c85b01 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 16 Sep 2025 17:15:49 +0200 Subject: [PATCH 11/87] add an exploratory test for quicker debugging of cache --- acceptance/cache/exploratory/databricks.yml | 2 ++ acceptance/cache/exploratory/out.test.toml | 5 +++++ acceptance/cache/exploratory/script | 7 +++++++ acceptance/cache/exploratory/test.toml | 2 ++ 4 files changed, 16 insertions(+) create mode 100644 acceptance/cache/exploratory/databricks.yml create mode 100644 acceptance/cache/exploratory/out.test.toml create mode 100644 acceptance/cache/exploratory/script create mode 100644 acceptance/cache/exploratory/test.toml diff --git a/acceptance/cache/exploratory/databricks.yml b/acceptance/cache/exploratory/databricks.yml new file mode 100644 index 00000000000..79d36f1e342 --- /dev/null +++ b/acceptance/cache/exploratory/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: exploratory-cache-test diff --git a/acceptance/cache/exploratory/out.test.toml b/acceptance/cache/exploratory/out.test.toml new file mode 100644 index 00000000000..f48015aedfe --- /dev/null +++ b/acceptance/cache/exploratory/out.test.toml @@ -0,0 +1,5 @@ +Local = false +Cloud = false + +[EnvMatrix] + DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"] diff --git a/acceptance/cache/exploratory/script b/acceptance/cache/exploratory/script new file mode 100644 index 00000000000..3e97a898e93 --- /dev/null +++ b/acceptance/cache/exploratory/script @@ -0,0 +1,7 @@ +unset DATABRICKS_CLIENT_SECRET +unset DATABRICKS_CLIENT_ID +unset DATABRICKS_HOST +unset DATABRICKS_AUTH_TYPE + +# export DATABRICKS_CONFIG_FILE=/Users//.databrickscfg +trace $CLI bundle validate -p dogfood diff --git a/acceptance/cache/exploratory/test.toml b/acceptance/cache/exploratory/test.toml new file mode 100644 index 00000000000..0902d334bd8 --- /dev/null +++ b/acceptance/cache/exploratory/test.toml @@ -0,0 +1,2 @@ +Cloud=false +Local=false From 6acaa442a21f6e7f0f0f657f1b7506c175e6d15a Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 16 Sep 2025 18:02:48 +0200 Subject: [PATCH 12/87] cleanup: send debugging statements to a separate log --- .../config/mutator/populate_current_user.go | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index 41bf6ad1fbe..876d564bee0 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -3,7 +3,7 @@ package mutator import ( "context" "encoding/json" - "fmt" + "github.com/databricks/cli/libs/log" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" @@ -36,7 +36,7 @@ func (m *populateCurrentUser) initializeCache(ctx context.Context, b *bundle.Bun m.cache = bundle.NewFileCache(cacheDir) - fmt.Printf("[DEBUG antonnek] New cache dir initialized: %s\n", cacheDir) + log.Debugf(ctx, "[Local Cache] New cache dir initialized: %s\n", cacheDir) return nil } @@ -52,11 +52,11 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. err := m.initializeCache(ctx, b) if err != nil { - fmt.Printf("[DEBUG antonnek] failed to initialize cache: %v \n", err) + log.Debugf(ctx, "[Local Cache] failed to initialize cache: %v \n", err) } w := b.WorkspaceClient() - bearerToken := m.getBearerToken(w) + bearerToken := m.getBearerToken(ctx, w) me := m.getUserFromCache(ctx, bearerToken) if me == nil { @@ -81,15 +81,15 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. } // getBearerToken extracts the bearer token from the workspace client's token source -func (m *populateCurrentUser) getBearerToken(w *databricks.WorkspaceClient) string { +func (m *populateCurrentUser) getBearerToken(ctx context.Context, w *databricks.WorkspaceClient) string { bearerToken := "" tokenSource := w.Config.GetTokenSource() if tokenSource == nil { - fmt.Printf("[DEBUG antonnek] token source not found\n") + log.Debugf(ctx, "[Local Cache] token source not found\n") } else { token, err := tokenSource.Token(context.Background()) if err != nil { - fmt.Printf("[DEBUG antonnek] error reading token source: %v \n", err) + log.Debugf(ctx, "[Local Cache] error reading token source: %v \n", err) } else { bearerToken = token.AccessToken } @@ -103,8 +103,8 @@ func (m *populateCurrentUser) getUserFromCache(ctx context.Context, bearerToken return nil } - fmt.Printf("[DEBUG antonnek] bearer token found: will use that for cache fingerprint\n") - fmt.Printf("[DEBUG antonnek] bearer token: %s\n", bearerToken) + log.Debugf(ctx, "[Local Cache] bearer token found: will use that for cache fingerprint\n") + log.Debugf(ctx, "[Local Cache] bearer token: %s\n", bearerToken) fingerprint, err := bundle.GenerateFingerprint("auth_header", bearerToken) if err != nil { @@ -115,7 +115,7 @@ func (m *populateCurrentUser) getUserFromCache(ctx context.Context, bearerToken if isCacheHit { var me *iam.User if err := json.Unmarshal(cachedUserBytes, &me); err == nil { - fmt.Printf("[DEBUG antonnek] user info read from cache: %s\n", fingerprint) + log.Debugf(ctx, "[Local Cache] user info read from cache: %s\n", fingerprint) return me } } @@ -131,7 +131,7 @@ func (m *populateCurrentUser) storeUserInCache(ctx context.Context, bearerToken userBytes, err := json.Marshal(user) if err != nil { - fmt.Printf("[DEBUG antonnek] could not serialize current user information: %v\n", err) + log.Debugf(ctx, "[Local Cache] could not serialize current user information: %v\n", err) return } @@ -142,8 +142,8 @@ func (m *populateCurrentUser) storeUserInCache(ctx context.Context, bearerToken err = m.cache.Store(ctx, fingerprint, userBytes) if err != nil { - fmt.Printf("[DEBUG antonnek] could not store user information: %v\n", err) + log.Debugf(ctx, "[Local Cache] could not store user information: %v\n", err) } else { - fmt.Printf("[DEBUG antonnek] stored user information in cache: %s\n", fingerprint) + log.Debugf(ctx, "[Local Cache] stored user information in cache: %s\n", fingerprint) } } From de89c9bfe88fbd5b3b5e34456de7b0221c7bb887 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 16 Sep 2025 18:07:47 +0200 Subject: [PATCH 13/87] make lint --- bundle/config/mutator/populate_current_user.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index 876d564bee0..5f5344a1924 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -3,6 +3,7 @@ package mutator import ( "context" "encoding/json" + "github.com/databricks/cli/libs/log" "github.com/databricks/cli/bundle" From e3a0239b9a804932f49d9823d2f73db3bd084a20 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 16 Sep 2025 18:18:12 +0200 Subject: [PATCH 14/87] revert populate_current_user.go --- .../config/mutator/populate_current_user.go | 113 +----------------- 1 file changed, 4 insertions(+), 109 deletions(-) diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index 5f5344a1924..1e7fae629e8 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -2,46 +2,21 @@ package mutator import ( "context" - "encoding/json" - - "github.com/databricks/cli/libs/log" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/iamutil" "github.com/databricks/cli/libs/tags" - "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/service/iam" ) -type populateCurrentUser struct { - cache bundle.Cache -} +type populateCurrentUser struct{} // PopulateCurrentUser sets the `current_user` property on the workspace. func PopulateCurrentUser() bundle.Mutator { return &populateCurrentUser{} } -// initializeCache sets up the cache for authorization headers if not already initialized -func (m *populateCurrentUser) initializeCache(ctx context.Context, b *bundle.Bundle) error { - if m.cache != nil { - return nil - } - - cacheDir, err := b.BundleLevelCacheDir(ctx, "auth") - if err != nil { - return err - } - - m.cache = bundle.NewFileCache(cacheDir) - - log.Debugf(ctx, "[Local Cache] New cache dir initialized: %s\n", cacheDir) - - return nil -} - func (m *populateCurrentUser) Name() string { return "PopulateCurrentUser" } @@ -51,22 +26,10 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. return nil } - err := m.initializeCache(ctx, b) - if err != nil { - log.Debugf(ctx, "[Local Cache] failed to initialize cache: %v \n", err) - } w := b.WorkspaceClient() - - bearerToken := m.getBearerToken(ctx, w) - me := m.getUserFromCache(ctx, bearerToken) - - if me == nil { - currentUser, err := w.CurrentUser.Me(ctx) - if err != nil { - return diag.FromErr(err) - } - me = currentUser - m.storeUserInCache(ctx, bearerToken, currentUser) + me, err := w.CurrentUser.Me(ctx) + if err != nil { + return diag.FromErr(err) } b.Config.Workspace.CurrentUser = &config.User{ @@ -80,71 +43,3 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. return nil } - -// getBearerToken extracts the bearer token from the workspace client's token source -func (m *populateCurrentUser) getBearerToken(ctx context.Context, w *databricks.WorkspaceClient) string { - bearerToken := "" - tokenSource := w.Config.GetTokenSource() - if tokenSource == nil { - log.Debugf(ctx, "[Local Cache] token source not found\n") - } else { - token, err := tokenSource.Token(context.Background()) - if err != nil { - log.Debugf(ctx, "[Local Cache] error reading token source: %v \n", err) - } else { - bearerToken = token.AccessToken - } - } - return bearerToken -} - -// getUserFromCache attempts to retrieve user information from cache using the bearer token -func (m *populateCurrentUser) getUserFromCache(ctx context.Context, bearerToken string) *iam.User { - if bearerToken == "" || m.cache == nil { - return nil - } - - log.Debugf(ctx, "[Local Cache] bearer token found: will use that for cache fingerprint\n") - log.Debugf(ctx, "[Local Cache] bearer token: %s\n", bearerToken) - - fingerprint, err := bundle.GenerateFingerprint("auth_header", bearerToken) - if err != nil { - panic(err) - } - - cachedUserBytes, isCacheHit := m.cache.Read(ctx, fingerprint) - if isCacheHit { - var me *iam.User - if err := json.Unmarshal(cachedUserBytes, &me); err == nil { - log.Debugf(ctx, "[Local Cache] user info read from cache: %s\n", fingerprint) - return me - } - } - - return nil -} - -// storeUserInCache stores user information in cache using the bearer token as key -func (m *populateCurrentUser) storeUserInCache(ctx context.Context, bearerToken string, user *iam.User) { - if bearerToken == "" || m.cache == nil { - return - } - - userBytes, err := json.Marshal(user) - if err != nil { - log.Debugf(ctx, "[Local Cache] could not serialize current user information: %v\n", err) - return - } - - fingerprint, err := bundle.GenerateFingerprint("auth_header", bearerToken) - if err != nil { - panic(err) - } - - err = m.cache.Store(ctx, fingerprint, userBytes) - if err != nil { - log.Debugf(ctx, "[Local Cache] could not store user information: %v\n", err) - } else { - log.Debugf(ctx, "[Local Cache] stored user information in cache: %s\n", fingerprint) - } -} From 6efc941b101a74c3db2f129196560002a61e52d5 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Thu, 18 Sep 2025 15:35:17 +0200 Subject: [PATCH 15/87] new Cache interface with a single GetOrCompute method --- bundle/cache.go | 19 ---- libs/cache/cache.go | 16 ++++ libs/cache/file_cache.go | 157 +++++++++++++++++++++++++++++++++ libs/cache/file_cache_test.go | 161 ++++++++++++++++++++++++++++++++++ 4 files changed, 334 insertions(+), 19 deletions(-) create mode 100644 libs/cache/cache.go create mode 100644 libs/cache/file_cache.go create mode 100644 libs/cache/file_cache_test.go diff --git a/bundle/cache.go b/bundle/cache.go index 1f64298b8d9..3d9e7ffc408 100644 --- a/bundle/cache.go +++ b/bundle/cache.go @@ -11,25 +11,6 @@ import ( "path/filepath" ) -// Cache provides an abstract interface for caching content to local disk. -// Implementations should handle storing and retrieving cached components -// using fingerprints for cache invalidation. -type Cache interface { - // Read retrieves cached content for the given fingerprint. - // Returns the cached data and true if found, or nil and false if not found or expired. - Read(ctx context.Context, fingerprint string) ([]byte, bool) - - // Store saves content to the cache with the given fingerprint. - // Returns an error if the cache operation fails. - Store(ctx context.Context, fingerprint string, content []byte) error - - // Clear removes all cached content from the cache directory. - Clear(ctx context.Context) error - - // ClearFingerprint removes cached content for a specific fingerprint. - ClearFingerprint(ctx context.Context, fingerprint string) error -} - // FileCache implements the Cache interface using the local filesystem. type FileCache struct { cachePath string diff --git a/libs/cache/cache.go b/libs/cache/cache.go new file mode 100644 index 00000000000..d2a4d9b7877 --- /dev/null +++ b/libs/cache/cache.go @@ -0,0 +1,16 @@ +package cache + +import ( + "context" +) + +// Cache provides an abstract interface for caching content to local disk. +// Implementations should handle storing and retrieving cached components +// using fingerprints for cache invalidation. +type Cache interface { + // GetOrCompute retrieves cached content for the given fingerprint, or computes it using the provided function. + // If the content is found in cache, it is returned directly. + // If not found, the compute function is called, its result is cached, and then returned. + // Returns an error if the cache operation or compute function fails. + GetOrCompute(ctx context.Context, fingerprint string, compute func(ctx context.Context) (any, error)) (any, error) +} diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go new file mode 100644 index 00000000000..3eb92fccd38 --- /dev/null +++ b/libs/cache/file_cache.go @@ -0,0 +1,157 @@ +package cache + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" + "time" +) + +// FileCache implements the Cache interface using local disk storage. +type FileCache struct { + baseDir string + mu sync.RWMutex + pending map[string]chan struct{} // Track pending writes +} + +// NewFileCache creates a new file-based cache that stores data in the specified directory. +func NewFileCache(baseDir string) (*FileCache, error) { + if err := os.MkdirAll(baseDir, 0755); err != nil { + return nil, fmt.Errorf("failed to create cache directory: %w", err) + } + + return &FileCache{ + baseDir: baseDir, + pending: make(map[string]chan struct{}), + }, nil +} + +// cacheEntry represents the structure of a cached item on disk. +type cacheEntry struct { + Data json.RawMessage `json:"data"` + Timestamp time.Time `json:"timestamp"` +} + +// GetOrCompute retrieves cached content or computes it using the provided function. +func (fc *FileCache) GetOrCompute(ctx context.Context, fingerprint string, compute func(ctx context.Context) (any, error)) (any, error) { + cacheKey := fc.getCacheKey(fingerprint) + cachePath := fc.getCachePath(cacheKey) + + // Try to read from cache first + if data, found := fc.readFromCache(cachePath); found { + return data, nil + } + + // Check if there's a pending write for this key + fc.mu.Lock() + if pendingCh, exists := fc.pending[cacheKey]; exists { + fc.mu.Unlock() + // Wait for pending write to complete + select { + case <-pendingCh: + // Try reading again after write completes + if data, found := fc.readFromCache(cachePath); found { + return data, nil + } + case <-ctx.Done(): + return nil, ctx.Err() + } + } else { + // Mark this key as pending + pendingCh := make(chan struct{}) + fc.pending[cacheKey] = pendingCh + fc.mu.Unlock() + + defer func() { + fc.mu.Lock() + delete(fc.pending, cacheKey) + close(pendingCh) + fc.mu.Unlock() + }() + } + + // Compute the value + result, err := compute(ctx) + if err != nil { + return nil, err + } + + // Async write to cache + go fc.writeToCache(cachePath, result) + + return result, nil +} + +// readFromCache attempts to read and deserialize data from the cache file. +func (fc *FileCache) readFromCache(cachePath string) (any, bool) { + fc.mu.RLock() + defer fc.mu.RUnlock() + + data, err := os.ReadFile(cachePath) + if err != nil { + return nil, false + } + + var entry cacheEntry + if err := json.Unmarshal(data, &entry); err != nil { + return nil, false + } + + var result any + if err := json.Unmarshal(entry.Data, &result); err != nil { + return nil, false + } + + return result, true +} + +// writeToCache serializes and writes data to the cache file asynchronously. +func (fc *FileCache) writeToCache(cachePath string, data any) { + // Serialize the data + serializedData, err := json.Marshal(data) + if err != nil { + return // Silently fail on serialization errors + } + + entry := cacheEntry{ + Data: serializedData, + Timestamp: time.Now(), + } + + entryData, err := json.Marshal(entry) + if err != nil { + return // Silently fail on serialization errors + } + + // Ensure directory exists + if err := os.MkdirAll(filepath.Dir(cachePath), 0755); err != nil { + return + } + + // Write to temporary file first, then rename for atomic operation + tempPath := cachePath + ".tmp" + if err := os.WriteFile(tempPath, entryData, 0644); err != nil { + return + } + + // Atomic rename + _ = os.Rename(tempPath, cachePath) +} + +// getCacheKey generates a safe cache key from the fingerprint. +func (fc *FileCache) getCacheKey(fingerprint string) string { + hash := sha256.Sum256([]byte(fingerprint)) + return hex.EncodeToString(hash[:]) +} + +// getCachePath returns the full path to the cache file for a given cache key. +func (fc *FileCache) getCachePath(cacheKey string) string { + // Create subdirectories based on first 2 characters for better file distribution + subDir := cacheKey[:2] + return filepath.Join(fc.baseDir, subDir, cacheKey+".json") +} diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go new file mode 100644 index 00000000000..fa2d110fa1b --- /dev/null +++ b/libs/cache/file_cache_test.go @@ -0,0 +1,161 @@ +package cache + +import ( + "context" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewFileCache(t *testing.T) { + tempDir := t.TempDir() + cacheDir := filepath.Join(tempDir, "cache") + + cache, err := NewFileCache(cacheDir) + require.NoError(t, err) + assert.NotNil(t, cache) + assert.Equal(t, cacheDir, cache.baseDir) + assert.NotNil(t, cache.pending) + + // Verify directory was created + info, err := os.Stat(cacheDir) + require.NoError(t, err) + assert.True(t, info.IsDir()) + assert.Equal(t, os.FileMode(0755), info.Mode().Perm()) +} + +func TestNewFileCacheWithExistingDirectory(t *testing.T) { + tempDir := t.TempDir() + cacheDir := filepath.Join(tempDir, "existing") + + // Create directory first + err := os.MkdirAll(cacheDir, 0700) + require.NoError(t, err) + + cache, err := NewFileCache(cacheDir) + require.NoError(t, err) + assert.NotNil(t, cache) + assert.Equal(t, cacheDir, cache.baseDir) +} + +func TestNewFileCacheInvalidPath(t *testing.T) { + // Try to create cache in a location that should fail + invalidPath := "/root/invalid/path/that/should/not/exist" + + cache, err := NewFileCache(invalidPath) + if err != nil { + assert.Nil(t, cache) + assert.Contains(t, err.Error(), "failed to create cache directory") + } +} + +func TestFileCacheGetOrCompute(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + cache, err := NewFileCache(tempDir) + require.NoError(t, err) + + fingerprint := "test-key" + expectedValue := "computed-value" + + // First call should compute the value + var computeCalls int32 + result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (any, error) { + atomic.AddInt32(&computeCalls, 1) + return expectedValue, nil + }) + + require.NoError(t, err) + assert.Equal(t, expectedValue, result) + assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) + + // Second call should return cached value + result2, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (any, error) { + atomic.AddInt32(&computeCalls, 1) + return "should-not-be-called", nil + }) + + require.NoError(t, err) + assert.Equal(t, expectedValue, result2) + assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) // Should still be 1 +} + +func TestFileCacheGetOrComputeError(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + cache, err := NewFileCache(tempDir) + require.NoError(t, err) + + fingerprint := "error-key" + + // Compute function returns error + result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (any, error) { + return nil, assert.AnError + }) + + assert.Nil(t, result) + assert.Error(t, err) + assert.Equal(t, assert.AnError, err) +} + +func TestFileCacheGetOrComputeConcurrency(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + cache, err := NewFileCache(tempDir) + require.NoError(t, err) + + fingerprint := "concurrent-key" + expectedValue := "concurrent-value" + var computeCalls int32 + + // Start multiple goroutines that try to compute the same key + numGoroutines := 10 + results := make(chan any, numGoroutines) + errors := make(chan error, numGoroutines) + + for i := 0; i < numGoroutines; i++ { + go func() { + result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (any, error) { + atomic.AddInt32(&computeCalls, 1) + time.Sleep(10 * time.Millisecond) // Simulate work + return expectedValue, nil + }) + results <- result + errors <- err + }() + } + + // Collect all results + for i := 0; i < numGoroutines; i++ { + result := <-results + err := <-errors + require.NoError(t, err) + assert.Equal(t, expectedValue, result) + } + + // Compute should have been called only once despite multiple concurrent requests + assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) +} + +func TestFileCacheGetOrComputeContextCancellation(t *testing.T) { + tempDir := t.TempDir() + cache, err := NewFileCache(tempDir) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Cancel immediately + + fingerprint := "cancelled-key" + + result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (any, error) { + return "should-not-be-reached", nil + }) + + assert.Nil(t, result) + assert.Equal(t, context.Canceled, err) +} From 9db3ee5fd0461c7e25ad1ccaf17540443c68c498 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Thu, 18 Sep 2025 15:47:35 +0200 Subject: [PATCH 16/87] unit tests are passing --- libs/cache/file_cache.go | 53 +++++++++++++++++++++++++++-------- libs/cache/file_cache_test.go | 8 +++--- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 3eb92fccd38..bfa9effbe56 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -14,20 +14,22 @@ import ( // FileCache implements the Cache interface using local disk storage. type FileCache struct { - baseDir string - mu sync.RWMutex - pending map[string]chan struct{} // Track pending writes + baseDir string + mu sync.RWMutex + pending map[string]chan struct{} // Track pending writes + memCache map[string]any // In-memory cache for immediate access } // NewFileCache creates a new file-based cache that stores data in the specified directory. func NewFileCache(baseDir string) (*FileCache, error) { - if err := os.MkdirAll(baseDir, 0755); err != nil { + if err := os.MkdirAll(baseDir, 0o755); err != nil { return nil, fmt.Errorf("failed to create cache directory: %w", err) } return &FileCache{ - baseDir: baseDir, - pending: make(map[string]chan struct{}), + baseDir: baseDir, + pending: make(map[string]chan struct{}), + memCache: make(map[string]any), }, nil } @@ -42,8 +44,20 @@ func (fc *FileCache) GetOrCompute(ctx context.Context, fingerprint string, compu cacheKey := fc.getCacheKey(fingerprint) cachePath := fc.getCachePath(cacheKey) - // Try to read from cache first + // Check in-memory cache first + fc.mu.RLock() + if data, found := fc.memCache[cacheKey]; found { + fc.mu.RUnlock() + return data, nil + } + fc.mu.RUnlock() + + // Try to read from disk cache if data, found := fc.readFromCache(cachePath); found { + // Store in memory cache for faster future access + fc.mu.Lock() + fc.memCache[cacheKey] = data + fc.mu.Unlock() return data, nil } @@ -54,10 +68,13 @@ func (fc *FileCache) GetOrCompute(ctx context.Context, fingerprint string, compu // Wait for pending write to complete select { case <-pendingCh: - // Try reading again after write completes - if data, found := fc.readFromCache(cachePath); found { + // Try reading from memory cache again + fc.mu.RLock() + if data, found := fc.memCache[cacheKey]; found { + fc.mu.RUnlock() return data, nil } + fc.mu.RUnlock() case <-ctx.Done(): return nil, ctx.Err() } @@ -75,13 +92,25 @@ func (fc *FileCache) GetOrCompute(ctx context.Context, fingerprint string, compu }() } + // Check if context is already cancelled before computing + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + // Compute the value result, err := compute(ctx) if err != nil { return nil, err } - // Async write to cache + // Store in memory cache immediately + fc.mu.Lock() + fc.memCache[cacheKey] = result + fc.mu.Unlock() + + // Async write to disk cache go fc.writeToCache(cachePath, result) return result, nil @@ -129,13 +158,13 @@ func (fc *FileCache) writeToCache(cachePath string, data any) { } // Ensure directory exists - if err := os.MkdirAll(filepath.Dir(cachePath), 0755); err != nil { + if err := os.MkdirAll(filepath.Dir(cachePath), 0o755); err != nil { return } // Write to temporary file first, then rename for atomic operation tempPath := cachePath + ".tmp" - if err := os.WriteFile(tempPath, entryData, 0644); err != nil { + if err := os.WriteFile(tempPath, entryData, 0o644); err != nil { return } diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index fa2d110fa1b..efc058f05d5 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -26,7 +26,7 @@ func TestNewFileCache(t *testing.T) { info, err := os.Stat(cacheDir) require.NoError(t, err) assert.True(t, info.IsDir()) - assert.Equal(t, os.FileMode(0755), info.Mode().Perm()) + assert.Equal(t, os.FileMode(0o755), info.Mode().Perm()) } func TestNewFileCacheWithExistingDirectory(t *testing.T) { @@ -34,7 +34,7 @@ func TestNewFileCacheWithExistingDirectory(t *testing.T) { cacheDir := filepath.Join(tempDir, "existing") // Create directory first - err := os.MkdirAll(cacheDir, 0700) + err := os.MkdirAll(cacheDir, 0o700) require.NoError(t, err) cache, err := NewFileCache(cacheDir) @@ -118,7 +118,7 @@ func TestFileCacheGetOrComputeConcurrency(t *testing.T) { results := make(chan any, numGoroutines) errors := make(chan error, numGoroutines) - for i := 0; i < numGoroutines; i++ { + for range numGoroutines { go func() { result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (any, error) { atomic.AddInt32(&computeCalls, 1) @@ -131,7 +131,7 @@ func TestFileCacheGetOrComputeConcurrency(t *testing.T) { } // Collect all results - for i := 0; i < numGoroutines; i++ { + for range numGoroutines { result := <-results err := <-errors require.NoError(t, err) From 0e47cf1b8597c398f506056c73ddaff603446973 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Thu, 18 Sep 2025 15:54:59 +0200 Subject: [PATCH 17/87] randomize temp path before writing cache to disk --- libs/cache/file_cache.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index bfa9effbe56..f67476338ad 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -2,6 +2,7 @@ package cache import ( "context" + "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" @@ -163,7 +164,11 @@ func (fc *FileCache) writeToCache(cachePath string, data any) { } // Write to temporary file first, then rename for atomic operation - tempPath := cachePath + ".tmp" + tempPath, err := generateTempPath(cachePath) + if err != nil { + return + } + if err := os.WriteFile(tempPath, entryData, 0o644); err != nil { return } @@ -172,6 +177,16 @@ func (fc *FileCache) writeToCache(cachePath string, data any) { _ = os.Rename(tempPath, cachePath) } +// generateTempPath creates a temporary file path with a random component to prevent collisions. +func generateTempPath(cachePath string) (string, error) { + randomBytes := make([]byte, 8) + if _, err := rand.Read(randomBytes); err != nil { + return "", err + } + randomSuffix := hex.EncodeToString(randomBytes) + return cachePath + ".tmp." + randomSuffix, nil +} + // getCacheKey generates a safe cache key from the fingerprint. func (fc *FileCache) getCacheKey(fingerprint string) string { hash := sha256.Sum256([]byte(fingerprint)) From e48ca3d99f62b01519f2d07fbc0cd8f6d2a9ee93 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Thu, 18 Sep 2025 16:30:08 +0200 Subject: [PATCH 18/87] use arbitrary structs as cache fingerprint --- libs/cache/cache.go | 66 ++++++++++++++++++++++++++- libs/cache/file_cache.go | 10 ++++- libs/cache/file_cache_test.go | 84 +++++++++++++++++++++++++++++++++-- 3 files changed, 153 insertions(+), 7 deletions(-) diff --git a/libs/cache/cache.go b/libs/cache/cache.go index d2a4d9b7877..916131f678d 100644 --- a/libs/cache/cache.go +++ b/libs/cache/cache.go @@ -2,6 +2,11 @@ package cache import ( "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "sort" ) // Cache provides an abstract interface for caching content to local disk. @@ -11,6 +16,65 @@ type Cache interface { // GetOrCompute retrieves cached content for the given fingerprint, or computes it using the provided function. // If the content is found in cache, it is returned directly. // If not found, the compute function is called, its result is cached, and then returned. + // The fingerprint can be any struct that will be serialized deterministically for cache key generation. // Returns an error if the cache operation or compute function fails. - GetOrCompute(ctx context.Context, fingerprint string, compute func(ctx context.Context) (any, error)) (any, error) + GetOrCompute(ctx context.Context, fingerprint any, compute func(ctx context.Context) (any, error)) (any, error) +} + +// FingerprintToString converts any struct to a deterministic string representation for use as a cache key. +func FingerprintToString(fingerprint any) (string, error) { + // Serialize to JSON with sorted keys for deterministic output + data, err := json.Marshal(fingerprint) + if err != nil { + return "", fmt.Errorf("failed to marshal fingerprint: %w", err) + } + + // Parse back to ensure consistent key ordering + var obj any + if err := json.Unmarshal(data, &obj); err != nil { + return "", fmt.Errorf("failed to unmarshal fingerprint: %w", err) + } + + // Sort keys deterministically + normalized := normalizeForFingerprint(obj) + + // Re-marshal with normalized structure + normalizedData, err := json.Marshal(normalized) + if err != nil { + return "", fmt.Errorf("failed to marshal normalized fingerprint: %w", err) + } + + // Hash the result for a consistent, reasonably-sized key + hash := sha256.Sum256(normalizedData) + return hex.EncodeToString(hash[:]), nil +} + +// normalizeForFingerprint recursively sorts map keys to ensure deterministic serialization. +func normalizeForFingerprint(obj any) any { + switch v := obj.(type) { + case map[string]any: + // Sort keys + keys := make([]string, 0, len(v)) + for k := range v { + keys = append(keys, k) + } + sort.Strings(keys) + + // Create ordered map + result := make(map[string]any, len(v)) + for _, k := range keys { + result[k] = normalizeForFingerprint(v[k]) + } + return result + case []any: + // Normalize each element in the slice + result := make([]any, len(v)) + for i, item := range v { + result[i] = normalizeForFingerprint(item) + } + return result + default: + // Primitive types are returned as-is + return v + } } diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index f67476338ad..3ad9eee3aaa 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -41,8 +41,14 @@ type cacheEntry struct { } // GetOrCompute retrieves cached content or computes it using the provided function. -func (fc *FileCache) GetOrCompute(ctx context.Context, fingerprint string, compute func(ctx context.Context) (any, error)) (any, error) { - cacheKey := fc.getCacheKey(fingerprint) +func (fc *FileCache) GetOrCompute(ctx context.Context, fingerprint any, compute func(ctx context.Context) (any, error)) (any, error) { + // Convert fingerprint to deterministic string + fingerprintStr, err := FingerprintToString(fingerprint) + if err != nil { + return nil, fmt.Errorf("failed to convert fingerprint to string: %w", err) + } + + cacheKey := fc.getCacheKey(fingerprintStr) cachePath := fc.getCachePath(cacheKey) // Check in-memory cache first diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index efc058f05d5..446e089a57c 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -60,7 +60,13 @@ func TestFileCacheGetOrCompute(t *testing.T) { cache, err := NewFileCache(tempDir) require.NoError(t, err) - fingerprint := "test-key" + fingerprint := struct { + Key string `json:"key"` + Value int `json:"value"` + }{ + Key: "test-key", + Value: 123, + } expectedValue := "computed-value" // First call should compute the value @@ -83,6 +89,9 @@ func TestFileCacheGetOrCompute(t *testing.T) { require.NoError(t, err) assert.Equal(t, expectedValue, result2) assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) // Should still be 1 + + // Allow time for async writes to complete before test cleanup + time.Sleep(50 * time.Millisecond) } func TestFileCacheGetOrComputeError(t *testing.T) { @@ -91,7 +100,11 @@ func TestFileCacheGetOrComputeError(t *testing.T) { cache, err := NewFileCache(tempDir) require.NoError(t, err) - fingerprint := "error-key" + fingerprint := struct { + Key string `json:"key"` + }{ + Key: "error-key", + } // Compute function returns error result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (any, error) { @@ -109,7 +122,11 @@ func TestFileCacheGetOrComputeConcurrency(t *testing.T) { cache, err := NewFileCache(tempDir) require.NoError(t, err) - fingerprint := "concurrent-key" + fingerprint := struct { + Key string `json:"key"` + }{ + Key: "concurrent-key", + } expectedValue := "concurrent-value" var computeCalls int32 @@ -140,6 +157,9 @@ func TestFileCacheGetOrComputeConcurrency(t *testing.T) { // Compute should have been called only once despite multiple concurrent requests assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) + + // Allow time for async writes to complete before test cleanup + time.Sleep(50 * time.Millisecond) } func TestFileCacheGetOrComputeContextCancellation(t *testing.T) { @@ -150,7 +170,11 @@ func TestFileCacheGetOrComputeContextCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() // Cancel immediately - fingerprint := "cancelled-key" + fingerprint := struct { + Key string `json:"key"` + }{ + Key: "cancelled-key", + } result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (any, error) { return "should-not-be-reached", nil @@ -159,3 +183,55 @@ func TestFileCacheGetOrComputeContextCancellation(t *testing.T) { assert.Nil(t, result) assert.Equal(t, context.Canceled, err) } + +func TestFingerprintDeterministic(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + cache, err := NewFileCache(tempDir) + require.NoError(t, err) + + // Create two identical structs with fields in different JSON order + fingerprint1 := struct { + A string `json:"a"` + B int `json:"b"` + C bool `json:"c"` + }{ + A: "value1", + B: 42, + C: true, + } + + fingerprint2 := struct { + C bool `json:"c"` + A string `json:"a"` + B int `json:"b"` + }{ + C: true, + A: "value1", + B: 42, + } + + expectedValue := "deterministic-value" + var computeCalls int32 + + // First call with fingerprint1 + result1, err := cache.GetOrCompute(ctx, fingerprint1, func(ctx context.Context) (any, error) { + atomic.AddInt32(&computeCalls, 1) + return expectedValue, nil + }) + require.NoError(t, err) + assert.Equal(t, expectedValue, result1) + assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) + + // Second call with fingerprint2 (should hit cache, not compute again) + result2, err := cache.GetOrCompute(ctx, fingerprint2, func(ctx context.Context) (any, error) { + atomic.AddInt32(&computeCalls, 1) + return "should-not-be-called", nil + }) + require.NoError(t, err) + assert.Equal(t, expectedValue, result2) + assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) // Should still be 1 + + // Allow time for async writes to complete before test cleanup + time.Sleep(50 * time.Millisecond) +} From a338c051e4b99214ac7c2a62d36f22cdd3865af6 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Thu, 18 Sep 2025 17:54:38 +0200 Subject: [PATCH 19/87] make fingerprintToString private --- libs/cache/cache.go | 4 ++-- libs/cache/file_cache.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/cache/cache.go b/libs/cache/cache.go index 916131f678d..bb6390f5ab2 100644 --- a/libs/cache/cache.go +++ b/libs/cache/cache.go @@ -21,8 +21,8 @@ type Cache interface { GetOrCompute(ctx context.Context, fingerprint any, compute func(ctx context.Context) (any, error)) (any, error) } -// FingerprintToString converts any struct to a deterministic string representation for use as a cache key. -func FingerprintToString(fingerprint any) (string, error) { +// fingerprintToString converts any struct to a deterministic string representation for use as a cache key. +func fingerprintToString(fingerprint any) (string, error) { // Serialize to JSON with sorted keys for deterministic output data, err := json.Marshal(fingerprint) if err != nil { diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 3ad9eee3aaa..94861b26a35 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -43,7 +43,7 @@ type cacheEntry struct { // GetOrCompute retrieves cached content or computes it using the provided function. func (fc *FileCache) GetOrCompute(ctx context.Context, fingerprint any, compute func(ctx context.Context) (any, error)) (any, error) { // Convert fingerprint to deterministic string - fingerprintStr, err := FingerprintToString(fingerprint) + fingerprintStr, err := fingerprintToString(fingerprint) if err != nil { return nil, fmt.Errorf("failed to convert fingerprint to string: %w", err) } From a37f176b0169449d468a91c9b71013e827dd330d Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 22 Sep 2025 13:47:22 +0200 Subject: [PATCH 20/87] make Cache interface generic --- .../mutator/populate_current_user_cached.go | 101 ++++++++++++++++++ libs/cache/cache.go | 8 +- libs/cache/file_cache.go | 64 +++++++---- libs/cache/file_cache_test.go | 36 +++---- 4 files changed, 164 insertions(+), 45 deletions(-) create mode 100644 bundle/config/mutator/populate_current_user_cached.go diff --git a/bundle/config/mutator/populate_current_user_cached.go b/bundle/config/mutator/populate_current_user_cached.go new file mode 100644 index 00000000000..321d1a85c05 --- /dev/null +++ b/bundle/config/mutator/populate_current_user_cached.go @@ -0,0 +1,101 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/libs/cache" + + "github.com/databricks/cli/libs/log" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/iamutil" + "github.com/databricks/cli/libs/tags" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/iam" +) + +type populateCurrentUserCached struct { + cache cache.Cache[*iam.User] +} + +// populateCurrentUserCached sets the `current_user` property on the workspace. +func PopulateCurrentUserCached() bundle.Mutator { + return &populateCurrentUserCached{} +} + +// initializeCache sets up the cache for authorization headers if not already initialized +func (m *populateCurrentUserCached) initializeCache(ctx context.Context, b *bundle.Bundle) error { + if m.cache != nil { + return nil + } + + cacheDir, err := b.BundleLevelCacheDir(ctx, "auth") + if err != nil { + return err + } + + m.cache, err = cache.NewFileCache[*iam.User](cacheDir) + if err != nil { + log.Debugf(ctx, "[Local Cache] Failed to initialize cache dir: %s\n", cacheDir) + } else { + log.Debugf(ctx, "[Local Cache] New cache dir initialized: %s\n", cacheDir) + } + + return nil +} + +func (m *populateCurrentUserCached) Name() string { + return "populateCurrentUserCached" +} + +func (m *populateCurrentUserCached) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if b.Config.Workspace.CurrentUser != nil { + return nil + } + + err := m.initializeCache(ctx, b) + if err != nil { + log.Debugf(ctx, "[Local Cache] failed to initialize cache: %v \n", err) + } + w := b.WorkspaceClient() + + bearerToken := m.getBearerToken(ctx, w) + + me, err := m.cache.GetOrCompute(ctx, bearerToken, func(ctx context.Context) (*iam.User, error) { + currentUser, err := w.CurrentUser.Me(ctx) + return currentUser, err + }) + if err != nil { + return diag.FromErr(err) + } + + b.Config.Workspace.CurrentUser = &config.User{ + ShortName: iamutil.GetShortUserName(me), + User: me, + } + + // Configure tagging object now that we know we have a valid client. + b.Tagging = tags.ForCloud(w.Config) + + return nil +} + +// getBearerToken extracts the bearer token from the workspace client's token source +func (m *populateCurrentUserCached) getBearerToken(ctx context.Context, w *databricks.WorkspaceClient) string { + bearerToken := "" + tokenSource := w.Config.GetTokenSource() + if tokenSource == nil { + log.Debugf(ctx, "[Local Cache] token source not found\n") + } else { + token, err := tokenSource.Token(context.Background()) + if err != nil { + log.Debugf(ctx, "[Local Cache] error reading token source: %v \n", err) + } else { + bearerToken = token.AccessToken + } + } + log.Debugf(ctx, "[Local Cache] found bearer token with length: %d\n", len(bearerToken)) + return bearerToken +} diff --git a/libs/cache/cache.go b/libs/cache/cache.go index bb6390f5ab2..d2931a270ce 100644 --- a/libs/cache/cache.go +++ b/libs/cache/cache.go @@ -12,17 +12,17 @@ import ( // Cache provides an abstract interface for caching content to local disk. // Implementations should handle storing and retrieving cached components // using fingerprints for cache invalidation. -type Cache interface { +type Cache[T any] interface { // GetOrCompute retrieves cached content for the given fingerprint, or computes it using the provided function. // If the content is found in cache, it is returned directly. // If not found, the compute function is called, its result is cached, and then returned. // The fingerprint can be any struct that will be serialized deterministically for cache key generation. // Returns an error if the cache operation or compute function fails. - GetOrCompute(ctx context.Context, fingerprint any, compute func(ctx context.Context) (any, error)) (any, error) + GetOrCompute(ctx context.Context, fingerprint any, compute func(ctx context.Context) (T, error)) (T, error) } -// fingerprintToString converts any struct to a deterministic string representation for use as a cache key. -func fingerprintToString(fingerprint any) (string, error) { +// fingerprintToHash converts any struct to a deterministic string representation for use as a cache key. +func fingerprintToHash(fingerprint any) (string, error) { // Serialize to JSON with sorted keys for deterministic output data, err := json.Marshal(fingerprint) if err != nil { diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 94861b26a35..b512424f131 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -11,26 +11,28 @@ import ( "path/filepath" "sync" "time" + + "github.com/databricks/cli/libs/log" ) // FileCache implements the Cache interface using local disk storage. -type FileCache struct { +type FileCache[T any] struct { baseDir string mu sync.RWMutex pending map[string]chan struct{} // Track pending writes - memCache map[string]any // In-memory cache for immediate access + memCache map[string]T // In-memory cache for immediate access } // NewFileCache creates a new file-based cache that stores data in the specified directory. -func NewFileCache(baseDir string) (*FileCache, error) { +func NewFileCache[T any](baseDir string) (*FileCache[T], error) { if err := os.MkdirAll(baseDir, 0o755); err != nil { return nil, fmt.Errorf("failed to create cache directory: %w", err) } - return &FileCache{ + return &FileCache[T]{ baseDir: baseDir, pending: make(map[string]chan struct{}), - memCache: make(map[string]any), + memCache: make(map[string]T), }, nil } @@ -41,20 +43,29 @@ type cacheEntry struct { } // GetOrCompute retrieves cached content or computes it using the provided function. -func (fc *FileCache) GetOrCompute(ctx context.Context, fingerprint any, compute func(ctx context.Context) (any, error)) (any, error) { +func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compute func(ctx context.Context) (T, error)) (T, error) { + var zero T + // Convert fingerprint to deterministic string - fingerprintStr, err := fingerprintToString(fingerprint) + fingerprintHash, err := fingerprintToHash(fingerprint) + log.Debugf(ctx, "[Local Cache] using fingerprint with hash: %s \n", fingerprintHash) + if err != nil { - return nil, fmt.Errorf("failed to convert fingerprint to string: %w", err) + log.Debugf(ctx, "[Local Cache] cache miss: non-compliant fingerprint \n") + return zero, fmt.Errorf("failed to convert fingerprint to string: %w", err) } - cacheKey := fc.getCacheKey(fingerprintStr) + cacheKey := fc.getCacheKey(fingerprintHash) + log.Debugf(ctx, "[Local Cache] using cache key: %s \n", cacheKey) + cachePath := fc.getCachePath(cacheKey) + log.Debugf(ctx, "[Local Cache] using cache path: %s \n", cachePath) // Check in-memory cache first fc.mu.RLock() if data, found := fc.memCache[cacheKey]; found { fc.mu.RUnlock() + log.Debugf(ctx, "[Local Cache] cache hit: in-memory \n") return data, nil } fc.mu.RUnlock() @@ -65,6 +76,7 @@ func (fc *FileCache) GetOrCompute(ctx context.Context, fingerprint any, compute fc.mu.Lock() fc.memCache[cacheKey] = data fc.mu.Unlock() + log.Debugf(ctx, "[Local Cache] cache hit: disk-read \n") return data, nil } @@ -79,11 +91,13 @@ func (fc *FileCache) GetOrCompute(ctx context.Context, fingerprint any, compute fc.mu.RLock() if data, found := fc.memCache[cacheKey]; found { fc.mu.RUnlock() + log.Debugf(ctx, "[Local Cache] cache hit: in-memory from pending write \n") return data, nil } fc.mu.RUnlock() case <-ctx.Done(): - return nil, ctx.Err() + log.Debugf(ctx, "[Local Cache] cache miss: no hit while waiting for pending write \n") + return zero, ctx.Err() } } else { // Mark this key as pending @@ -102,14 +116,16 @@ func (fc *FileCache) GetOrCompute(ctx context.Context, fingerprint any, compute // Check if context is already cancelled before computing select { case <-ctx.Done(): - return nil, ctx.Err() + log.Debugf(ctx, "[Local Cache] cache miss: context is already cancelled \n") + return zero, ctx.Err() default: } // Compute the value result, err := compute(ctx) if err != nil { - return nil, err + log.Debugf(ctx, "[Local Cache] error while caching: %v \n", err) + return zero, err } // Store in memory cache immediately @@ -118,36 +134,40 @@ func (fc *FileCache) GetOrCompute(ctx context.Context, fingerprint any, compute fc.mu.Unlock() // Async write to disk cache + log.Debugf(ctx, "[Local Cache] async writing to cache: %s :: %v \n", cachePath, result) go fc.writeToCache(cachePath, result) + log.Debugf(ctx, "[Local Cache] cache miss, but stored the compute result for future calls \n") return result, nil } // readFromCache attempts to read and deserialize data from the cache file. -func (fc *FileCache) readFromCache(cachePath string) (any, bool) { +func (fc *FileCache[T]) readFromCache(cachePath string) (T, bool) { + var zero T + fc.mu.RLock() defer fc.mu.RUnlock() data, err := os.ReadFile(cachePath) if err != nil { - return nil, false + return zero, false } var entry cacheEntry if err := json.Unmarshal(data, &entry); err != nil { - return nil, false + return zero, false } - var result any + var result T if err := json.Unmarshal(entry.Data, &result); err != nil { - return nil, false + return zero, false } return result, true } // writeToCache serializes and writes data to the cache file asynchronously. -func (fc *FileCache) writeToCache(cachePath string, data any) { +func (fc *FileCache[T]) writeToCache(cachePath string, data any) { // Serialize the data serializedData, err := json.Marshal(data) if err != nil { @@ -194,14 +214,12 @@ func generateTempPath(cachePath string) (string, error) { } // getCacheKey generates a safe cache key from the fingerprint. -func (fc *FileCache) getCacheKey(fingerprint string) string { +func (fc *FileCache[T]) getCacheKey(fingerprint string) string { hash := sha256.Sum256([]byte(fingerprint)) return hex.EncodeToString(hash[:]) } // getCachePath returns the full path to the cache file for a given cache key. -func (fc *FileCache) getCachePath(cacheKey string) string { - // Create subdirectories based on first 2 characters for better file distribution - subDir := cacheKey[:2] - return filepath.Join(fc.baseDir, subDir, cacheKey+".json") +func (fc *FileCache[T]) getCachePath(cacheKey string) string { + return filepath.Join(fc.baseDir, cacheKey+".json") } diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index 446e089a57c..2222996ef5d 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -16,7 +16,7 @@ func TestNewFileCache(t *testing.T) { tempDir := t.TempDir() cacheDir := filepath.Join(tempDir, "cache") - cache, err := NewFileCache(cacheDir) + cache, err := NewFileCache[string](cacheDir) require.NoError(t, err) assert.NotNil(t, cache) assert.Equal(t, cacheDir, cache.baseDir) @@ -37,7 +37,7 @@ func TestNewFileCacheWithExistingDirectory(t *testing.T) { err := os.MkdirAll(cacheDir, 0o700) require.NoError(t, err) - cache, err := NewFileCache(cacheDir) + cache, err := NewFileCache[string](cacheDir) require.NoError(t, err) assert.NotNil(t, cache) assert.Equal(t, cacheDir, cache.baseDir) @@ -47,7 +47,7 @@ func TestNewFileCacheInvalidPath(t *testing.T) { // Try to create cache in a location that should fail invalidPath := "/root/invalid/path/that/should/not/exist" - cache, err := NewFileCache(invalidPath) + cache, err := NewFileCache[string](invalidPath) if err != nil { assert.Nil(t, cache) assert.Contains(t, err.Error(), "failed to create cache directory") @@ -57,7 +57,7 @@ func TestNewFileCacheInvalidPath(t *testing.T) { func TestFileCacheGetOrCompute(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := NewFileCache(tempDir) + cache, err := NewFileCache[string](tempDir) require.NoError(t, err) fingerprint := struct { @@ -71,7 +71,7 @@ func TestFileCacheGetOrCompute(t *testing.T) { // First call should compute the value var computeCalls int32 - result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (any, error) { + result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return expectedValue, nil }) @@ -81,7 +81,7 @@ func TestFileCacheGetOrCompute(t *testing.T) { assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) // Second call should return cached value - result2, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (any, error) { + result2, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "should-not-be-called", nil }) @@ -97,7 +97,7 @@ func TestFileCacheGetOrCompute(t *testing.T) { func TestFileCacheGetOrComputeError(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := NewFileCache(tempDir) + cache, err := NewFileCache[string](tempDir) require.NoError(t, err) fingerprint := struct { @@ -107,11 +107,11 @@ func TestFileCacheGetOrComputeError(t *testing.T) { } // Compute function returns error - result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (any, error) { - return nil, assert.AnError + result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + return "", assert.AnError }) - assert.Nil(t, result) + assert.Empty(t, result) assert.Error(t, err) assert.Equal(t, assert.AnError, err) } @@ -119,7 +119,7 @@ func TestFileCacheGetOrComputeError(t *testing.T) { func TestFileCacheGetOrComputeConcurrency(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := NewFileCache(tempDir) + cache, err := NewFileCache[string](tempDir) require.NoError(t, err) fingerprint := struct { @@ -137,7 +137,7 @@ func TestFileCacheGetOrComputeConcurrency(t *testing.T) { for range numGoroutines { go func() { - result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (any, error) { + result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) time.Sleep(10 * time.Millisecond) // Simulate work return expectedValue, nil @@ -164,7 +164,7 @@ func TestFileCacheGetOrComputeConcurrency(t *testing.T) { func TestFileCacheGetOrComputeContextCancellation(t *testing.T) { tempDir := t.TempDir() - cache, err := NewFileCache(tempDir) + cache, err := NewFileCache[string](tempDir) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -176,18 +176,18 @@ func TestFileCacheGetOrComputeContextCancellation(t *testing.T) { Key: "cancelled-key", } - result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (any, error) { + result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { return "should-not-be-reached", nil }) - assert.Nil(t, result) + assert.Empty(t, result) assert.Equal(t, context.Canceled, err) } func TestFingerprintDeterministic(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := NewFileCache(tempDir) + cache, err := NewFileCache[string](tempDir) require.NoError(t, err) // Create two identical structs with fields in different JSON order @@ -215,7 +215,7 @@ func TestFingerprintDeterministic(t *testing.T) { var computeCalls int32 // First call with fingerprint1 - result1, err := cache.GetOrCompute(ctx, fingerprint1, func(ctx context.Context) (any, error) { + result1, err := cache.GetOrCompute(ctx, fingerprint1, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return expectedValue, nil }) @@ -224,7 +224,7 @@ func TestFingerprintDeterministic(t *testing.T) { assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) // Second call with fingerprint2 (should hit cache, not compute again) - result2, err := cache.GetOrCompute(ctx, fingerprint2, func(ctx context.Context) (any, error) { + result2, err := cache.GetOrCompute(ctx, fingerprint2, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "should-not-be-called", nil }) From e5669aac97996ada3b86d22996f4ce70c082cbca Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 22 Sep 2025 14:59:42 +0200 Subject: [PATCH 21/87] temporarily use `DATABRICKS_EXPERIMENTAL_CACHE_ENABLED` env var to enable caching in populate_current_user_cached --- .../mutator/populate_current_user_cached.go | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/bundle/config/mutator/populate_current_user_cached.go b/bundle/config/mutator/populate_current_user_cached.go index 321d1a85c05..e929b83d04b 100644 --- a/bundle/config/mutator/populate_current_user_cached.go +++ b/bundle/config/mutator/populate_current_user_cached.go @@ -2,6 +2,7 @@ package mutator import ( "context" + "os" "github.com/databricks/cli/libs/cache" @@ -26,14 +27,20 @@ func PopulateCurrentUserCached() bundle.Mutator { } // initializeCache sets up the cache for authorization headers if not already initialized -func (m *populateCurrentUserCached) initializeCache(ctx context.Context, b *bundle.Bundle) error { +func (m *populateCurrentUserCached) initializeCache(ctx context.Context, b *bundle.Bundle) { if m.cache != nil { - return nil + return + } + + if os.Getenv("DATABRICKS_EXPERIMENTAL_CACHE_ENABLED") != "true" { + log.Debugf(ctx, "[Local Cache] Local cache is disabled. Enable it be setting an env variable DATABRICKS_EXPERIMENTAL_CACHE_ENABLED=true \n") + return } cacheDir, err := b.BundleLevelCacheDir(ctx, "auth") if err != nil { - return err + log.Debugf(ctx, "[Local Cache] BundleLevelCacheDir could not initialize: %v \n", err) + return } m.cache, err = cache.NewFileCache[*iam.User](cacheDir) @@ -42,8 +49,6 @@ func (m *populateCurrentUserCached) initializeCache(ctx context.Context, b *bund } else { log.Debugf(ctx, "[Local Cache] New cache dir initialized: %s\n", cacheDir) } - - return nil } func (m *populateCurrentUserCached) Name() string { @@ -54,23 +59,33 @@ func (m *populateCurrentUserCached) Apply(ctx context.Context, b *bundle.Bundle) if b.Config.Workspace.CurrentUser != nil { return nil } - - err := m.initializeCache(ctx, b) - if err != nil { - log.Debugf(ctx, "[Local Cache] failed to initialize cache: %v \n", err) - } + m.initializeCache(ctx, b) w := b.WorkspaceClient() bearerToken := m.getBearerToken(ctx, w) - me, err := m.cache.GetOrCompute(ctx, bearerToken, func(ctx context.Context) (*iam.User, error) { - currentUser, err := w.CurrentUser.Me(ctx) - return currentUser, err - }) + var me *iam.User + var err error + + if m.cache != nil { + log.Debugf(ctx, "[Local Cache] local cache is enabled \n") + me, err = m.cache.GetOrCompute(ctx, bearerToken, func(ctx context.Context) (*iam.User, error) { + currentUser, err := w.CurrentUser.Me(ctx) + return currentUser, err + }) + } else { + log.Debugf(ctx, "[Local Cache] local cache is disabled \n") + me, err = w.CurrentUser.Me(ctx) + } + if err != nil { return diag.FromErr(err) } + if me == nil { + return diag.Errorf("could not find current user, but no error was returned") + } + b.Config.Workspace.CurrentUser = &config.User{ ShortName: iamutil.GetShortUserName(me), User: me, From e097a73d7bb27021d5b94e3295dac410b2072627 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 22 Sep 2025 15:02:43 +0200 Subject: [PATCH 22/87] remove bundle/cache.go --- bundle/cache.go | 101 ------------------------------------------------ 1 file changed, 101 deletions(-) delete mode 100644 bundle/cache.go diff --git a/bundle/cache.go b/bundle/cache.go deleted file mode 100644 index 3d9e7ffc408..00000000000 --- a/bundle/cache.go +++ /dev/null @@ -1,101 +0,0 @@ -package bundle - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "os" - "path/filepath" -) - -// FileCache implements the Cache interface using the local filesystem. -type FileCache struct { - cachePath string -} - -// NewFileCache creates a new filesystem-based cache at the specified path. -func NewFileCache(cachePath string) *FileCache { - return &FileCache{ - cachePath: cachePath, - } -} - -// Read retrieves cached content for the given fingerprint. -func (fc *FileCache) Read(ctx context.Context, fingerprint string) ([]byte, bool) { - filePath := fc.getFilePath(fingerprint) - data, err := os.ReadFile(filePath) - if err != nil { - return nil, false - } - - return data, true -} - -// Store saves content to the cache with the given fingerprint. -func (fc *FileCache) Store(ctx context.Context, fingerprint string, content []byte) error { - filePath := fc.getFilePath(fingerprint) - if err := os.WriteFile(filePath, content, 0o600); err != nil { - return fmt.Errorf("failed to write cache file: %w", err) - } - - return nil -} - -// Clear removes all cached content from the cache directory. -func (fc *FileCache) Clear(ctx context.Context) error { - if _, err := os.Stat(fc.cachePath); os.IsNotExist(err) { - return nil - } - - return os.RemoveAll(fc.cachePath) -} - -// ClearFingerprint removes cached content for a specific fingerprint. -func (fc *FileCache) ClearFingerprint(ctx context.Context, fingerprint string) error { - filePath := fc.getFilePath(fingerprint) - if err := os.Remove(filePath); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to remove cache file: %w", err) - } - return nil -} - -// getFilePath returns the full file path for a given fingerprint. -func (fc *FileCache) getFilePath(fingerprint string) string { - return filepath.Join(fc.cachePath, fingerprint+".cache") -} - -// GenerateFingerprint creates a SHA256 fingerprint from the provided data. -// This is a utility function for creating consistent fingerprints. -func GenerateFingerprint(data ...any) (string, error) { - hasher := sha256.New() - - for _, item := range data { - var bytes []byte - var err error - - switch v := item.(type) { - case string: - bytes = []byte(v) - case []byte: - bytes = v - case io.Reader: - bytes, err = io.ReadAll(v) - if err != nil { - return "", fmt.Errorf("failed to read data for fingerprint: %w", err) - } - default: - bytes, err = json.Marshal(v) - if err != nil { - return "", fmt.Errorf("failed to marshal data for fingerprint: %w", err) - } - } - - hasher.Write(bytes) - } - - hash := hasher.Sum(nil) - return hex.EncodeToString(hash[:16]), nil -} From 75d51be4fc01764f90c46fe726a258b02c2d9580 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 22 Sep 2025 15:26:29 +0200 Subject: [PATCH 23/87] use struct as a fingerprint --- bundle/config/mutator/populate_current_user_cached.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bundle/config/mutator/populate_current_user_cached.go b/bundle/config/mutator/populate_current_user_cached.go index e929b83d04b..8b802154990 100644 --- a/bundle/config/mutator/populate_current_user_cached.go +++ b/bundle/config/mutator/populate_current_user_cached.go @@ -62,14 +62,18 @@ func (m *populateCurrentUserCached) Apply(ctx context.Context, b *bundle.Bundle) m.initializeCache(ctx, b) w := b.WorkspaceClient() - bearerToken := m.getBearerToken(ctx, w) + fingerprint := struct { + bearerToken string + }{ + bearerToken: m.getBearerToken(ctx, w), + } var me *iam.User var err error if m.cache != nil { log.Debugf(ctx, "[Local Cache] local cache is enabled \n") - me, err = m.cache.GetOrCompute(ctx, bearerToken, func(ctx context.Context) (*iam.User, error) { + me, err = m.cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (*iam.User, error) { currentUser, err := w.CurrentUser.Me(ctx) return currentUser, err }) From 8309318542e6d4e99a15271a13f2dd31aa53dad6 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 22 Sep 2025 15:51:13 +0200 Subject: [PATCH 24/87] use os.UserCacheDir as the root dir for cache files --- .../mutator/populate_current_user_cached.go | 17 +++++------------ libs/cache/file_cache.go | 15 +++++++++++++-- libs/cache/file_cache_test.go | 16 ++++++++-------- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/bundle/config/mutator/populate_current_user_cached.go b/bundle/config/mutator/populate_current_user_cached.go index 8b802154990..9195e354572 100644 --- a/bundle/config/mutator/populate_current_user_cached.go +++ b/bundle/config/mutator/populate_current_user_cached.go @@ -27,7 +27,7 @@ func PopulateCurrentUserCached() bundle.Mutator { } // initializeCache sets up the cache for authorization headers if not already initialized -func (m *populateCurrentUserCached) initializeCache(ctx context.Context, b *bundle.Bundle) { +func (m *populateCurrentUserCached) initializeCache(ctx context.Context) { if m.cache != nil { return } @@ -37,17 +37,10 @@ func (m *populateCurrentUserCached) initializeCache(ctx context.Context, b *bund return } - cacheDir, err := b.BundleLevelCacheDir(ctx, "auth") - if err != nil { - log.Debugf(ctx, "[Local Cache] BundleLevelCacheDir could not initialize: %v \n", err) - return - } - - m.cache, err = cache.NewFileCache[*iam.User](cacheDir) + var err error + m.cache, err = cache.NewFileCache[*iam.User]("auth") if err != nil { - log.Debugf(ctx, "[Local Cache] Failed to initialize cache dir: %s\n", cacheDir) - } else { - log.Debugf(ctx, "[Local Cache] New cache dir initialized: %s\n", cacheDir) + log.Debugf(ctx, "[Local Cache] Failed to initialize cache dir: %v \n", err) } } @@ -59,7 +52,7 @@ func (m *populateCurrentUserCached) Apply(ctx context.Context, b *bundle.Bundle) if b.Config.Workspace.CurrentUser != nil { return nil } - m.initializeCache(ctx, b) + m.initializeCache(ctx) w := b.WorkspaceClient() fingerprint := struct { diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index b512424f131..60747a42ad0 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -23,8 +23,8 @@ type FileCache[T any] struct { memCache map[string]T // In-memory cache for immediate access } -// NewFileCache creates a new file-based cache that stores data in the specified directory. -func NewFileCache[T any](baseDir string) (*FileCache[T], error) { +// newFileCacheWithBaseDir creates a new file-based cache that stores data in the specified directory. +func newFileCacheWithBaseDir[T any](baseDir string) (*FileCache[T], error) { if err := os.MkdirAll(baseDir, 0o755); err != nil { return nil, fmt.Errorf("failed to create cache directory: %w", err) } @@ -36,6 +36,17 @@ func NewFileCache[T any](baseDir string) (*FileCache[T], error) { }, nil } +// NewFileCache creates a new file-based cache using UserCacheDir() + "databricks" + cached component name. +func NewFileCache[T any](component string) (*FileCache[T], error) { + userCacheDir, err := os.UserCacheDir() + if err != nil { + return nil, fmt.Errorf("failed to get user cache directory: %w", err) + } + + baseDir := filepath.Join(userCacheDir, "databricks", component) + return newFileCacheWithBaseDir[T](baseDir) +} + // cacheEntry represents the structure of a cached item on disk. type cacheEntry struct { Data json.RawMessage `json:"data"` diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index 2222996ef5d..2e303d74327 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -16,7 +16,7 @@ func TestNewFileCache(t *testing.T) { tempDir := t.TempDir() cacheDir := filepath.Join(tempDir, "cache") - cache, err := NewFileCache[string](cacheDir) + cache, err := newFileCacheWithBaseDir[string](cacheDir) require.NoError(t, err) assert.NotNil(t, cache) assert.Equal(t, cacheDir, cache.baseDir) @@ -37,7 +37,7 @@ func TestNewFileCacheWithExistingDirectory(t *testing.T) { err := os.MkdirAll(cacheDir, 0o700) require.NoError(t, err) - cache, err := NewFileCache[string](cacheDir) + cache, err := newFileCacheWithBaseDir[string](cacheDir) require.NoError(t, err) assert.NotNil(t, cache) assert.Equal(t, cacheDir, cache.baseDir) @@ -47,7 +47,7 @@ func TestNewFileCacheInvalidPath(t *testing.T) { // Try to create cache in a location that should fail invalidPath := "/root/invalid/path/that/should/not/exist" - cache, err := NewFileCache[string](invalidPath) + cache, err := newFileCacheWithBaseDir[string](invalidPath) if err != nil { assert.Nil(t, cache) assert.Contains(t, err.Error(), "failed to create cache directory") @@ -57,7 +57,7 @@ func TestNewFileCacheInvalidPath(t *testing.T) { func TestFileCacheGetOrCompute(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := NewFileCache[string](tempDir) + cache, err := newFileCacheWithBaseDir[string](tempDir) require.NoError(t, err) fingerprint := struct { @@ -97,7 +97,7 @@ func TestFileCacheGetOrCompute(t *testing.T) { func TestFileCacheGetOrComputeError(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := NewFileCache[string](tempDir) + cache, err := newFileCacheWithBaseDir[string](tempDir) require.NoError(t, err) fingerprint := struct { @@ -119,7 +119,7 @@ func TestFileCacheGetOrComputeError(t *testing.T) { func TestFileCacheGetOrComputeConcurrency(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := NewFileCache[string](tempDir) + cache, err := newFileCacheWithBaseDir[string](tempDir) require.NoError(t, err) fingerprint := struct { @@ -164,7 +164,7 @@ func TestFileCacheGetOrComputeConcurrency(t *testing.T) { func TestFileCacheGetOrComputeContextCancellation(t *testing.T) { tempDir := t.TempDir() - cache, err := NewFileCache[string](tempDir) + cache, err := newFileCacheWithBaseDir[string](tempDir) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -187,7 +187,7 @@ func TestFileCacheGetOrComputeContextCancellation(t *testing.T) { func TestFingerprintDeterministic(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := NewFileCache[string](tempDir) + cache, err := newFileCacheWithBaseDir[string](tempDir) require.NoError(t, err) // Create two identical structs with fields in different JSON order From f0849fd3bd2ea1bdfde97792017bf79615cd7707 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 22 Sep 2025 15:55:32 +0200 Subject: [PATCH 25/87] remove BundleLevelCacheDir --- bundle/bundle.go | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/bundle/bundle.go b/bundle/bundle.go index a2f2a03f86c..efe7cad9a6f 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -290,36 +290,6 @@ func (b *Bundle) InternalDir(ctx context.Context) (string, error) { return dir, nil } -// BundleLevelCacheDir is used to cache components needed for the bundle that are target-independent -func (b *Bundle) BundleLevelCacheDir(ctx context.Context, cacheComponentName string) (string, error) { - cacheDirName, exists := env.TempDir(ctx) - if !exists || cacheDirName == "" { - cacheDirName = filepath.Join( - // Anchor at bundle root directory. - b.BundleRootPath, - // Static cache directory. - ".databricks", - ) - } - - // Fixed components of the result path. - parts := []string{ - cacheDirName, - cacheFolder, - cacheComponentName, - } - - // Make directory if it doesn't exist yet. - dir := filepath.Join(parts...) - err := os.MkdirAll(dir, 0o700) - if err != nil { - return "", err - } - - libsync.WriteGitIgnore(ctx, b.BundleRootPath) - return dir, nil -} - // GetSyncIncludePatterns returns a list of user defined includes // And also adds InternalDir folder to include list for sync command // so this folder is always synced From 06e9d5611d8314f2a763b3993d61c2cf0cec8873 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 22 Sep 2025 16:22:44 +0200 Subject: [PATCH 26/87] fix unit test on windows --- libs/cache/file_cache_test.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index 2e303d74327..b6dad9eb04d 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "runtime" "sync/atomic" "testing" "time" @@ -26,7 +27,19 @@ func TestNewFileCache(t *testing.T) { info, err := os.Stat(cacheDir) require.NoError(t, err) assert.True(t, info.IsDir()) - assert.Equal(t, os.FileMode(0o755), info.Mode().Perm()) + + // Check permissions - Windows has different permission semantics + if runtime.GOOS != "windows" { + assert.Equal(t, os.FileMode(0o755), info.Mode().Perm()) + } else { + // On Windows, verify directory is accessible by trying to create a test file + testFile := filepath.Join(cacheDir, "test_access") + err := os.WriteFile(testFile, []byte("test"), 0o644) + assert.NoError(t, err) + if err == nil { + _ = os.Remove(testFile) // Clean up (ignore removal error) + } + } } func TestNewFileCacheWithExistingDirectory(t *testing.T) { From ebefd4353e0d517caa6eeca72148da51fef68f0a Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 22 Sep 2025 16:36:27 +0200 Subject: [PATCH 27/87] remove unused variable --- bundle/bundle.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bundle/bundle.go b/bundle/bundle.go index efe7cad9a6f..e34012580c7 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -32,10 +32,7 @@ import ( "github.com/hashicorp/terraform-exec/tfexec" ) -const ( - internalFolder = ".internal" - cacheFolder = ".cache" -) +const internalFolder = ".internal" // Filename where resources are stored for DATABRICKS_BUNDLE_ENGINE=direct const resourcesFilename = "resources.json" From c60339be0fd7dec5143cd84143bb206e2824d13f Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 23 Sep 2025 13:32:24 +0200 Subject: [PATCH 28/87] do not use cache if bearer token is empty --- acceptance/cache/exploratory/script | 12 +++++++----- .../config/mutator/populate_current_user_cached.go | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/acceptance/cache/exploratory/script b/acceptance/cache/exploratory/script index 3e97a898e93..32eaedf616a 100644 --- a/acceptance/cache/exploratory/script +++ b/acceptance/cache/exploratory/script @@ -1,7 +1,9 @@ -unset DATABRICKS_CLIENT_SECRET -unset DATABRICKS_CLIENT_ID -unset DATABRICKS_HOST -unset DATABRICKS_AUTH_TYPE +#unset DATABRICKS_CLIENT_SECRET +#unset DATABRICKS_CLIENT_ID +#unset DATABRICKS_HOST +#unset DATABRICKS_AUTH_TYPE # export DATABRICKS_CONFIG_FILE=/Users//.databrickscfg -trace $CLI bundle validate -p dogfood + +export DATABRICKS_EXPERIMENTAL_CACHE_ENABLED=true +trace $CLI bundle validate -p dogfood --debug diff --git a/bundle/config/mutator/populate_current_user_cached.go b/bundle/config/mutator/populate_current_user_cached.go index 9195e354572..00cf990da71 100644 --- a/bundle/config/mutator/populate_current_user_cached.go +++ b/bundle/config/mutator/populate_current_user_cached.go @@ -64,7 +64,7 @@ func (m *populateCurrentUserCached) Apply(ctx context.Context, b *bundle.Bundle) var me *iam.User var err error - if m.cache != nil { + if m.cache != nil && fingerprint.bearerToken != "" { log.Debugf(ctx, "[Local Cache] local cache is enabled \n") me, err = m.cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (*iam.User, error) { currentUser, err := w.CurrentUser.Me(ctx) From 7dcca0fc7b9e3d5695b6824b457018228c86c49d Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 23 Sep 2025 14:33:44 +0200 Subject: [PATCH 29/87] use auth header as a cache fingerprint --- acceptance/cache/exploratory/out.test.toml | 4 +-- acceptance/cache/exploratory/output.txt | 14 +++++++++ acceptance/cache/exploratory/script | 12 +++---- acceptance/cache/exploratory/test.toml | 10 +++++- .../mutator/populate_current_user_cached.go | 31 ++++++++----------- libs/cache/file_cache.go | 2 +- 6 files changed, 44 insertions(+), 29 deletions(-) create mode 100644 acceptance/cache/exploratory/output.txt diff --git a/acceptance/cache/exploratory/out.test.toml b/acceptance/cache/exploratory/out.test.toml index f48015aedfe..e092fd5ed6a 100644 --- a/acceptance/cache/exploratory/out.test.toml +++ b/acceptance/cache/exploratory/out.test.toml @@ -1,5 +1,5 @@ -Local = false +Local = true Cloud = false [EnvMatrix] - DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct-exp"] diff --git a/acceptance/cache/exploratory/output.txt b/acceptance/cache/exploratory/output.txt new file mode 100644 index 00000000000..48534d916d3 --- /dev/null +++ b/acceptance/cache/exploratory/output.txt @@ -0,0 +1,14 @@ + +=== First call in a session is expected to be a cache miss: +[DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 +[DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled +[DEBUG_TIMESTAMP] Debug: [Local Cache] using fingerprint with hash: [SHA256_HASH] +[DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, but stored the compute result for future calls + +=== Second call in a session is expected to be a cache hit +[DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 +[DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled +[DEBUG_TIMESTAMP] Debug: [Local Cache] using fingerprint with hash: [SHA256_HASH] +[DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit: disk-read diff --git a/acceptance/cache/exploratory/script b/acceptance/cache/exploratory/script index 32eaedf616a..9c225d79684 100644 --- a/acceptance/cache/exploratory/script +++ b/acceptance/cache/exploratory/script @@ -1,9 +1,7 @@ -#unset DATABRICKS_CLIENT_SECRET -#unset DATABRICKS_CLIENT_ID -#unset DATABRICKS_HOST -#unset DATABRICKS_AUTH_TYPE +export DATABRICKS_EXPERIMENTAL_CACHE_ENABLED=true -# export DATABRICKS_CONFIG_FILE=/Users//.databrickscfg +title "First call in a session is expected to be a cache miss:\n" +trace $CLI bundle validate -p dogfood --debug 2>&1 | grep "Local Cache" | grep -v "cache path" -export DATABRICKS_EXPERIMENTAL_CACHE_ENABLED=true -trace $CLI bundle validate -p dogfood --debug +title "Second call in a session is expected to be a cache hit\n" +trace $CLI bundle validate -p dogfood --debug 2>&1 | grep "Local Cache" | grep -v "cache path" diff --git a/acceptance/cache/exploratory/test.toml b/acceptance/cache/exploratory/test.toml index 0902d334bd8..2853ed8cf72 100644 --- a/acceptance/cache/exploratory/test.toml +++ b/acceptance/cache/exploratory/test.toml @@ -1,2 +1,10 @@ Cloud=false -Local=false +Local=true + +[[Repls]] +Old = '\d\d:\d\d:\d\d' +New = "[DEBUG_TIMESTAMP]" + +[[Repls]] +Old = '[a-f0-9]{64}' +New = "[SHA256_HASH]" diff --git a/bundle/config/mutator/populate_current_user_cached.go b/bundle/config/mutator/populate_current_user_cached.go index 00cf990da71..f4d8cc64774 100644 --- a/bundle/config/mutator/populate_current_user_cached.go +++ b/bundle/config/mutator/populate_current_user_cached.go @@ -2,6 +2,7 @@ package mutator import ( "context" + "net/http" "os" "github.com/databricks/cli/libs/cache" @@ -56,15 +57,15 @@ func (m *populateCurrentUserCached) Apply(ctx context.Context, b *bundle.Bundle) w := b.WorkspaceClient() fingerprint := struct { - bearerToken string + authHeader string }{ - bearerToken: m.getBearerToken(ctx, w), + authHeader: m.getAuthorizationHeader(ctx, w), } var me *iam.User var err error - if m.cache != nil && fingerprint.bearerToken != "" { + if m.cache != nil && fingerprint.authHeader != "" { log.Debugf(ctx, "[Local Cache] local cache is enabled \n") me, err = m.cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (*iam.User, error) { currentUser, err := w.CurrentUser.Me(ctx) @@ -94,20 +95,14 @@ func (m *populateCurrentUserCached) Apply(ctx context.Context, b *bundle.Bundle) return nil } -// getBearerToken extracts the bearer token from the workspace client's token source -func (m *populateCurrentUserCached) getBearerToken(ctx context.Context, w *databricks.WorkspaceClient) string { - bearerToken := "" - tokenSource := w.Config.GetTokenSource() - if tokenSource == nil { - log.Debugf(ctx, "[Local Cache] token source not found\n") - } else { - token, err := tokenSource.Token(context.Background()) - if err != nil { - log.Debugf(ctx, "[Local Cache] error reading token source: %v \n", err) - } else { - bearerToken = token.AccessToken - } +func (m *populateCurrentUserCached) getAuthorizationHeader(ctx context.Context, w *databricks.WorkspaceClient) string { + // Create a dummy request to extract the Authorization header + req := &http.Request{Header: http.Header{}} + if err := w.Config.Authenticate(req); err != nil { + return "" } - log.Debugf(ctx, "[Local Cache] found bearer token with length: %d\n", len(bearerToken)) - return bearerToken + + authHeader := req.Header.Get("Authorization") + log.Debugf(ctx, "[Local Cache] found authorization header with length: %d\n", len(authHeader)) + return authHeader } diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 60747a42ad0..4e27cd9106b 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -145,7 +145,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu fc.mu.Unlock() // Async write to disk cache - log.Debugf(ctx, "[Local Cache] async writing to cache: %s :: %v \n", cachePath, result) + log.Debugf(ctx, "[Local Cache] async writing to cache path: %s \n", cachePath) go fc.writeToCache(cachePath, result) log.Debugf(ctx, "[Local Cache] cache miss, but stored the compute result for future calls \n") From 4b3a48bd1f3dc5578152f145550d890f44a65607 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 23 Sep 2025 14:42:24 +0200 Subject: [PATCH 30/87] fix trailing whitespaces in debug messages --- acceptance/cache/exploratory/output.txt | 16 ++++++------- .../mutator/populate_current_user_cached.go | 8 +++---- libs/cache/file_cache.go | 24 +++++++++---------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/acceptance/cache/exploratory/output.txt b/acceptance/cache/exploratory/output.txt index 48534d916d3..b2bec4c8d85 100644 --- a/acceptance/cache/exploratory/output.txt +++ b/acceptance/cache/exploratory/output.txt @@ -1,14 +1,14 @@ === First call in a session is expected to be a cache miss: [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 -[DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled -[DEBUG_TIMESTAMP] Debug: [Local Cache] using fingerprint with hash: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, but stored the compute result for future calls +[DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled +[DEBUG_TIMESTAMP] Debug: [Local Cache] using fingerprint with hash: [SHA256_HASH] +[DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, but stored the compute result for future calls === Second call in a session is expected to be a cache hit [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 -[DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled -[DEBUG_TIMESTAMP] Debug: [Local Cache] using fingerprint with hash: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit: disk-read +[DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled +[DEBUG_TIMESTAMP] Debug: [Local Cache] using fingerprint with hash: [SHA256_HASH] +[DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit: disk-read diff --git a/bundle/config/mutator/populate_current_user_cached.go b/bundle/config/mutator/populate_current_user_cached.go index f4d8cc64774..7755f943c82 100644 --- a/bundle/config/mutator/populate_current_user_cached.go +++ b/bundle/config/mutator/populate_current_user_cached.go @@ -34,14 +34,14 @@ func (m *populateCurrentUserCached) initializeCache(ctx context.Context) { } if os.Getenv("DATABRICKS_EXPERIMENTAL_CACHE_ENABLED") != "true" { - log.Debugf(ctx, "[Local Cache] Local cache is disabled. Enable it be setting an env variable DATABRICKS_EXPERIMENTAL_CACHE_ENABLED=true \n") + log.Debugf(ctx, "[Local Cache] Local cache is disabled. Enable it be setting an env variable DATABRICKS_EXPERIMENTAL_CACHE_ENABLED=true\n") return } var err error m.cache, err = cache.NewFileCache[*iam.User]("auth") if err != nil { - log.Debugf(ctx, "[Local Cache] Failed to initialize cache dir: %v \n", err) + log.Debugf(ctx, "[Local Cache] Failed to initialize cache dir: %v\n", err) } } @@ -66,13 +66,13 @@ func (m *populateCurrentUserCached) Apply(ctx context.Context, b *bundle.Bundle) var err error if m.cache != nil && fingerprint.authHeader != "" { - log.Debugf(ctx, "[Local Cache] local cache is enabled \n") + log.Debugf(ctx, "[Local Cache] local cache is enabled\n") me, err = m.cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (*iam.User, error) { currentUser, err := w.CurrentUser.Me(ctx) return currentUser, err }) } else { - log.Debugf(ctx, "[Local Cache] local cache is disabled \n") + log.Debugf(ctx, "[Local Cache] local cache is disabled\n") me, err = w.CurrentUser.Me(ctx) } diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 4e27cd9106b..79d3c8568ec 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -59,24 +59,24 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu // Convert fingerprint to deterministic string fingerprintHash, err := fingerprintToHash(fingerprint) - log.Debugf(ctx, "[Local Cache] using fingerprint with hash: %s \n", fingerprintHash) + log.Debugf(ctx, "[Local Cache] using fingerprint with hash: %s\n", fingerprintHash) if err != nil { - log.Debugf(ctx, "[Local Cache] cache miss: non-compliant fingerprint \n") + log.Debugf(ctx, "[Local Cache] cache miss: non-compliant fingerprint\n") return zero, fmt.Errorf("failed to convert fingerprint to string: %w", err) } cacheKey := fc.getCacheKey(fingerprintHash) - log.Debugf(ctx, "[Local Cache] using cache key: %s \n", cacheKey) + log.Debugf(ctx, "[Local Cache] using cache key: %s\n", cacheKey) cachePath := fc.getCachePath(cacheKey) - log.Debugf(ctx, "[Local Cache] using cache path: %s \n", cachePath) + log.Debugf(ctx, "[Local Cache] using cache path: %s\n", cachePath) // Check in-memory cache first fc.mu.RLock() if data, found := fc.memCache[cacheKey]; found { fc.mu.RUnlock() - log.Debugf(ctx, "[Local Cache] cache hit: in-memory \n") + log.Debugf(ctx, "[Local Cache] cache hit: in-memory\n") return data, nil } fc.mu.RUnlock() @@ -87,7 +87,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu fc.mu.Lock() fc.memCache[cacheKey] = data fc.mu.Unlock() - log.Debugf(ctx, "[Local Cache] cache hit: disk-read \n") + log.Debugf(ctx, "[Local Cache] cache hit: disk-read\n") return data, nil } @@ -102,12 +102,12 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu fc.mu.RLock() if data, found := fc.memCache[cacheKey]; found { fc.mu.RUnlock() - log.Debugf(ctx, "[Local Cache] cache hit: in-memory from pending write \n") + log.Debugf(ctx, "[Local Cache] cache hit: in-memory from pending write\n") return data, nil } fc.mu.RUnlock() case <-ctx.Done(): - log.Debugf(ctx, "[Local Cache] cache miss: no hit while waiting for pending write \n") + log.Debugf(ctx, "[Local Cache] cache miss: no hit while waiting for pending write\n") return zero, ctx.Err() } } else { @@ -127,7 +127,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu // Check if context is already cancelled before computing select { case <-ctx.Done(): - log.Debugf(ctx, "[Local Cache] cache miss: context is already cancelled \n") + log.Debugf(ctx, "[Local Cache] cache miss: context is already cancelled\n") return zero, ctx.Err() default: } @@ -135,7 +135,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu // Compute the value result, err := compute(ctx) if err != nil { - log.Debugf(ctx, "[Local Cache] error while caching: %v \n", err) + log.Debugf(ctx, "[Local Cache] error while caching: %v\n", err) return zero, err } @@ -145,10 +145,10 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu fc.mu.Unlock() // Async write to disk cache - log.Debugf(ctx, "[Local Cache] async writing to cache path: %s \n", cachePath) + log.Debugf(ctx, "[Local Cache] async writing to cache path: %s\n", cachePath) go fc.writeToCache(cachePath, result) - log.Debugf(ctx, "[Local Cache] cache miss, but stored the compute result for future calls \n") + log.Debugf(ctx, "[Local Cache] cache miss, but stored the compute result for future calls\n") return result, nil } From aea6b1fb9529db8fce4d88b6467a3be097f03b7e Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 23 Sep 2025 14:59:19 +0200 Subject: [PATCH 31/87] replace populate_current_user.go with the implementation that has a cache layer --- .../config/mutator/populate_current_user.go | 75 +++++++++++- .../mutator/populate_current_user_cached.go | 108 ------------------ 2 files changed, 69 insertions(+), 114 deletions(-) delete mode 100644 bundle/config/mutator/populate_current_user_cached.go diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index 1e7fae629e8..ac64c979700 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -2,21 +2,49 @@ package mutator import ( "context" + "net/http" + "os" + + "github.com/databricks/cli/libs/cache" + + "github.com/databricks/cli/libs/log" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/iamutil" "github.com/databricks/cli/libs/tags" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/iam" ) -type populateCurrentUser struct{} +type populateCurrentUser struct { + cache cache.Cache[*iam.User] +} // PopulateCurrentUser sets the `current_user` property on the workspace. func PopulateCurrentUser() bundle.Mutator { return &populateCurrentUser{} } +// initializeCache sets up the cache for authorization headers if not already initialized +func (m *populateCurrentUser) initializeCache(ctx context.Context) { + if m.cache != nil { + return + } + + if os.Getenv("DATABRICKS_EXPERIMENTAL_CACHE_ENABLED") != "true" { + log.Debugf(ctx, "[Local Cache] Local cache is disabled. Enable it be setting an env variable DATABRICKS_EXPERIMENTAL_CACHE_ENABLED=true\n") + return + } + + var err error + m.cache, err = cache.NewFileCache[*iam.User]("auth") + if err != nil { + log.Debugf(ctx, "[Local Cache] Failed to initialize cache dir: %v\n", err) + } +} + func (m *populateCurrentUser) Name() string { return "PopulateCurrentUser" } @@ -25,17 +53,40 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. if b.Config.Workspace.CurrentUser != nil { return nil } - + m.initializeCache(ctx) w := b.WorkspaceClient() - me, err := w.CurrentUser.Me(ctx) + + fingerprint := struct { + authHeader string + }{ + authHeader: m.getAuthorizationHeader(ctx, w), + } + + var me *iam.User + var err error + + if m.cache != nil && fingerprint.authHeader != "" { + log.Debugf(ctx, "[Local Cache] local cache is enabled\n") + me, err = m.cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (*iam.User, error) { + currentUser, err := w.CurrentUser.Me(ctx) + return currentUser, err + }) + } else { + log.Debugf(ctx, "[Local Cache] local cache is disabled\n") + me, err = w.CurrentUser.Me(ctx) + } + if err != nil { return diag.FromErr(err) } + if me == nil { + return diag.Errorf("could not find current user, but no error was returned") + } + b.Config.Workspace.CurrentUser = &config.User{ - ShortName: iamutil.GetShortUserName(me), - DomainFriendlyName: iamutil.GetShortUserDomainFriendlyName(me), - User: me, + ShortName: iamutil.GetShortUserName(me), + User: me, } // Configure tagging object now that we know we have a valid client. @@ -43,3 +94,15 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. return nil } + +func (m *populateCurrentUser) getAuthorizationHeader(ctx context.Context, w *databricks.WorkspaceClient) string { + // Create a dummy request to extract the Authorization header + req := &http.Request{Header: http.Header{}} + if err := w.Config.Authenticate(req); err != nil { + return "" + } + + authHeader := req.Header.Get("Authorization") + log.Debugf(ctx, "[Local Cache] found authorization header with length: %d\n", len(authHeader)) + return authHeader +} diff --git a/bundle/config/mutator/populate_current_user_cached.go b/bundle/config/mutator/populate_current_user_cached.go deleted file mode 100644 index 7755f943c82..00000000000 --- a/bundle/config/mutator/populate_current_user_cached.go +++ /dev/null @@ -1,108 +0,0 @@ -package mutator - -import ( - "context" - "net/http" - "os" - - "github.com/databricks/cli/libs/cache" - - "github.com/databricks/cli/libs/log" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/iamutil" - "github.com/databricks/cli/libs/tags" - "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/service/iam" -) - -type populateCurrentUserCached struct { - cache cache.Cache[*iam.User] -} - -// populateCurrentUserCached sets the `current_user` property on the workspace. -func PopulateCurrentUserCached() bundle.Mutator { - return &populateCurrentUserCached{} -} - -// initializeCache sets up the cache for authorization headers if not already initialized -func (m *populateCurrentUserCached) initializeCache(ctx context.Context) { - if m.cache != nil { - return - } - - if os.Getenv("DATABRICKS_EXPERIMENTAL_CACHE_ENABLED") != "true" { - log.Debugf(ctx, "[Local Cache] Local cache is disabled. Enable it be setting an env variable DATABRICKS_EXPERIMENTAL_CACHE_ENABLED=true\n") - return - } - - var err error - m.cache, err = cache.NewFileCache[*iam.User]("auth") - if err != nil { - log.Debugf(ctx, "[Local Cache] Failed to initialize cache dir: %v\n", err) - } -} - -func (m *populateCurrentUserCached) Name() string { - return "populateCurrentUserCached" -} - -func (m *populateCurrentUserCached) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - if b.Config.Workspace.CurrentUser != nil { - return nil - } - m.initializeCache(ctx) - w := b.WorkspaceClient() - - fingerprint := struct { - authHeader string - }{ - authHeader: m.getAuthorizationHeader(ctx, w), - } - - var me *iam.User - var err error - - if m.cache != nil && fingerprint.authHeader != "" { - log.Debugf(ctx, "[Local Cache] local cache is enabled\n") - me, err = m.cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (*iam.User, error) { - currentUser, err := w.CurrentUser.Me(ctx) - return currentUser, err - }) - } else { - log.Debugf(ctx, "[Local Cache] local cache is disabled\n") - me, err = w.CurrentUser.Me(ctx) - } - - if err != nil { - return diag.FromErr(err) - } - - if me == nil { - return diag.Errorf("could not find current user, but no error was returned") - } - - b.Config.Workspace.CurrentUser = &config.User{ - ShortName: iamutil.GetShortUserName(me), - User: me, - } - - // Configure tagging object now that we know we have a valid client. - b.Tagging = tags.ForCloud(w.Config) - - return nil -} - -func (m *populateCurrentUserCached) getAuthorizationHeader(ctx context.Context, w *databricks.WorkspaceClient) string { - // Create a dummy request to extract the Authorization header - req := &http.Request{Header: http.Header{}} - if err := w.Config.Authenticate(req); err != nil { - return "" - } - - authHeader := req.Header.Get("Authorization") - log.Debugf(ctx, "[Local Cache] found authorization header with length: %d\n", len(authHeader)) - return authHeader -} From 12671941fead69652a7fb1a0c2bccba3468bf311 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 23 Sep 2025 15:05:13 +0200 Subject: [PATCH 32/87] restore DomainFriendlyName --- bundle/config/mutator/populate_current_user.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index ac64c979700..4207fed9029 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -85,8 +85,9 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. } b.Config.Workspace.CurrentUser = &config.User{ - ShortName: iamutil.GetShortUserName(me), - User: me, + ShortName: iamutil.GetShortUserName(me), + DomainFriendlyName: iamutil.GetShortUserDomainFriendlyName(me), + User: me, } // Configure tagging object now that we know we have a valid client. From 45c8234c19bc98ac5b9f609a363c9fb472d39e05 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 23 Sep 2025 15:37:57 +0200 Subject: [PATCH 33/87] basic cache cleanup in background --- libs/cache/cleanup.go | 196 ++++++++++++++++++++++ libs/cache/cleanup_test.go | 321 +++++++++++++++++++++++++++++++++++++ libs/cache/file_cache.go | 35 ++-- 3 files changed, 543 insertions(+), 9 deletions(-) create mode 100644 libs/cache/cleanup.go create mode 100644 libs/cache/cleanup_test.go diff --git a/libs/cache/cleanup.go b/libs/cache/cleanup.go new file mode 100644 index 00000000000..df9527b44af --- /dev/null +++ b/libs/cache/cleanup.go @@ -0,0 +1,196 @@ +package cache + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/databricks/cli/libs/log" +) + +// CleanupConfig holds configuration for cache cleanup. +type CleanupConfig struct { + MaxAge time.Duration // Maximum age of cache files before cleanup + ScanInterval time.Duration // How often to scan for old files + DryRun bool // If true, only logs what would be deleted +} + +// DefaultCleanupConfig returns sensible defaults for cache cleanup. +func DefaultCleanupConfig() CleanupConfig { + return CleanupConfig{ + MaxAge: 7 * 24 * time.Hour, // 7 days + ScanInterval: 24 * time.Hour, // Daily cleanup + DryRun: false, + } +} + +// CleanupManager manages background cleanup of cache files. +type CleanupManager struct { + config CleanupConfig + stopCh chan struct{} + stoppedCh chan struct{} + mu sync.Mutex + running bool + stopped bool +} + +// NewCleanupManager creates a new cleanup manager with the given configuration. +func NewCleanupManager(config CleanupConfig) *CleanupManager { + return &CleanupManager{ + config: config, + stopCh: make(chan struct{}), + stoppedCh: make(chan struct{}), + } +} + +// Start begins the background cleanup process. +// This is non-blocking and will not prevent the main process from exiting. +func (cm *CleanupManager) Start(ctx context.Context, cacheDir string) { + cm.mu.Lock() + defer cm.mu.Unlock() + + if cm.running || cm.stopped { + return // Already running or stopped + } + + cm.running = true + + go func() { + defer func() { + cm.mu.Lock() + cm.running = false + cm.mu.Unlock() + close(cm.stoppedCh) + }() + + log.Debugf(ctx, "[Cache Cleanup] Starting cleanup manager for directory: %s", cacheDir) + + // Perform initial cleanup + cm.cleanup(ctx, cacheDir) + + // Set up periodic cleanup + ticker := time.NewTicker(cm.config.ScanInterval) + defer ticker.Stop() + + for { + select { + case <-cm.stopCh: + log.Debugf(ctx, "[Cache Cleanup] Cleanup manager stopped") + return + case <-ticker.C: + cm.cleanup(ctx, cacheDir) + } + } + }() +} + +// Stop gracefully stops the cleanup manager. +// This is non-blocking and returns immediately. +func (cm *CleanupManager) Stop() { + cm.mu.Lock() + defer cm.mu.Unlock() + + if !cm.running || cm.stopped { + return + } + + cm.stopped = true + close(cm.stopCh) +} + +// Wait waits for the cleanup manager to stop completely. +// This should only be used in tests or shutdown scenarios where you need to wait. +func (cm *CleanupManager) Wait() { + <-cm.stoppedCh +} + +// cleanup performs the actual cleanup of old cache files. +func (cm *CleanupManager) cleanup(ctx context.Context, cacheDir string) { + log.Debugf(ctx, "[Cache Cleanup] Starting cleanup scan of directory: %s", cacheDir) + + // Check if cache directory exists + if _, err := os.Stat(cacheDir); os.IsNotExist(err) { + log.Debugf(ctx, "[Cache Cleanup] Cache directory does not exist: %s", cacheDir) + return + } + + var deletedCount, scannedCount int + var totalSize, deletedSize int64 + cutoff := time.Now().Add(-cm.config.MaxAge) + + err := filepath.Walk(cacheDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + log.Debugf(ctx, "[Cache Cleanup] Error accessing path %s: %v", path, err) + return nil // Continue with other files + } + + // Skip directories and non-cache files + if info.IsDir() || !strings.HasSuffix(info.Name(), ".json") { + return nil + } + + scannedCount++ + totalSize += info.Size() + + shouldDelete, fileAge := cm.shouldDeleteFile(ctx, path, cutoff) + if shouldDelete { + deletedSize += info.Size() + deletedCount++ + + if cm.config.DryRun { + log.Debugf(ctx, "[Cache Cleanup] Would delete old cache file: %s (age: %v)", path, fileAge) + } else { + if err := os.Remove(path); err != nil { + log.Debugf(ctx, "[Cache Cleanup] Failed to delete cache file %s: %v", path, err) + } else { + log.Debugf(ctx, "[Cache Cleanup] Deleted old cache file: %s (age: %v)", path, fileAge) + } + } + } + + return nil + }) + if err != nil { + log.Debugf(ctx, "[Cache Cleanup] Error during cleanup scan: %v", err) + } + + action := "deleted" + if cm.config.DryRun { + action = "would delete" + } + + log.Debugf(ctx, "[Cache Cleanup] Cleanup complete: scanned %d files (%.2f MB), %s %d files (%.2f MB)", + scannedCount, float64(totalSize)/(1024*1024), + action, deletedCount, float64(deletedSize)/(1024*1024)) +} + +// shouldDeleteFile determines if a cache file should be deleted based on its age. +func (cm *CleanupManager) shouldDeleteFile(ctx context.Context, path string, cutoff time.Time) (bool, time.Duration) { + // Try to read the cache entry to get the timestamp + data, err := os.ReadFile(path) + if err != nil { + // If we can't read the file, use file modification time as fallback + if info, statErr := os.Stat(path); statErr == nil { + age := time.Since(info.ModTime()) + return info.ModTime().Before(cutoff), age + } + return true, time.Duration(0) // Delete unreadable files + } + + var entry cacheEntry + if err := json.Unmarshal(data, &entry); err != nil { + // If we can't parse the cache entry, use file modification time as fallback + if info, statErr := os.Stat(path); statErr == nil { + age := time.Since(info.ModTime()) + return info.ModTime().Before(cutoff), age + } + return true, time.Duration(0) // Delete unparseable files + } + + age := time.Since(entry.Timestamp) + return entry.Timestamp.Before(cutoff), age +} diff --git a/libs/cache/cleanup_test.go b/libs/cache/cleanup_test.go new file mode 100644 index 00000000000..51244999de0 --- /dev/null +++ b/libs/cache/cleanup_test.go @@ -0,0 +1,321 @@ +package cache + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDefaultCleanupConfig(t *testing.T) { + config := DefaultCleanupConfig() + assert.Equal(t, 7*24*time.Hour, config.MaxAge) + assert.Equal(t, 24*time.Hour, config.ScanInterval) + assert.False(t, config.DryRun) +} + +func TestCleanupManager_Start_Stop(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + + config := CleanupConfig{ + MaxAge: time.Hour, + ScanInterval: 100 * time.Millisecond, + DryRun: false, + } + + manager := NewCleanupManager(config) + + // Start the manager + manager.Start(ctx, tempDir) + assert.True(t, manager.running) + + // Stop the manager + manager.Stop() + + // Wait for it to stop with timeout + done := make(chan struct{}) + go func() { + manager.Wait() + close(done) + }() + + select { + case <-done: + assert.False(t, manager.running) + case <-time.After(5 * time.Second): + t.Fatal("Cleanup manager did not stop within timeout") + } +} + +func TestCleanupManager_CleanupOldFiles(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + + config := CleanupConfig{ + MaxAge: time.Hour, + ScanInterval: time.Hour, // Long interval to prevent automatic cleanup during test + DryRun: false, + } + + manager := NewCleanupManager(config) + + // Create test files with different ages + now := time.Now() + + // Create an old file (should be deleted) + oldFile := filepath.Join(tempDir, "old_file.json") + oldEntry := cacheEntry{ + Data: json.RawMessage(`"old_data"`), + Timestamp: now.Add(-2 * time.Hour), // 2 hours old + } + oldData, err := json.Marshal(oldEntry) + require.NoError(t, err) + require.NoError(t, os.WriteFile(oldFile, oldData, 0o644)) + + // Create a recent file (should not be deleted) + recentFile := filepath.Join(tempDir, "recent_file.json") + recentEntry := cacheEntry{ + Data: json.RawMessage(`"recent_data"`), + Timestamp: now.Add(-30 * time.Minute), // 30 minutes old + } + recentData, err := json.Marshal(recentEntry) + require.NoError(t, err) + require.NoError(t, os.WriteFile(recentFile, recentData, 0o644)) + + // Create a non-cache file (should be ignored) + nonCacheFile := filepath.Join(tempDir, "not_cache.txt") + require.NoError(t, os.WriteFile(nonCacheFile, []byte("not cache"), 0o644)) + + // Run cleanup manually + manager.cleanup(ctx, tempDir) + + // Check results + _, err = os.Stat(oldFile) + assert.True(t, os.IsNotExist(err), "Old file should be deleted") + + _, err = os.Stat(recentFile) + assert.False(t, os.IsNotExist(err), "Recent file should not be deleted") + + _, err = os.Stat(nonCacheFile) + assert.False(t, os.IsNotExist(err), "Non-cache file should not be deleted") +} + +func TestCleanupManager_DryRun(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + + config := CleanupConfig{ + MaxAge: time.Hour, + ScanInterval: time.Hour, + DryRun: true, // Dry run mode + } + + manager := NewCleanupManager(config) + + // Create an old file + now := time.Now() + oldFile := filepath.Join(tempDir, "old_file.json") + oldEntry := cacheEntry{ + Data: json.RawMessage(`"old_data"`), + Timestamp: now.Add(-2 * time.Hour), + } + oldData, err := json.Marshal(oldEntry) + require.NoError(t, err) + require.NoError(t, os.WriteFile(oldFile, oldData, 0o644)) + + // Run cleanup in dry run mode + manager.cleanup(ctx, tempDir) + + // File should still exist in dry run mode + _, err = os.Stat(oldFile) + assert.False(t, os.IsNotExist(err), "File should not be deleted in dry run mode") +} + +func TestCleanupManager_CorruptedFiles(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + + config := CleanupConfig{ + MaxAge: time.Hour, + ScanInterval: time.Hour, + DryRun: false, + } + + manager := NewCleanupManager(config) + + // Create a corrupted cache file (invalid JSON) + corruptedFile := filepath.Join(tempDir, "corrupted.json") + require.NoError(t, os.WriteFile(corruptedFile, []byte("invalid json"), 0o644)) + + // Set old modification time + oldTime := time.Now().Add(-2 * time.Hour) + require.NoError(t, os.Chtimes(corruptedFile, oldTime, oldTime)) + + // Create a file with invalid cache entry structure + invalidStructureFile := filepath.Join(tempDir, "invalid_structure.json") + require.NoError(t, os.WriteFile(invalidStructureFile, []byte(`{"invalid": "structure"}`), 0o644)) + + // Set old modification time + require.NoError(t, os.Chtimes(invalidStructureFile, oldTime, oldTime)) + + // Run cleanup - corrupted files should be deleted + manager.cleanup(ctx, tempDir) + + // Both corrupted files should be deleted + _, err := os.Stat(corruptedFile) + assert.True(t, os.IsNotExist(err), "Corrupted file should be deleted") + + _, err = os.Stat(invalidStructureFile) + assert.True(t, os.IsNotExist(err), "Invalid structure file should be deleted") +} + +func TestCleanupManager_NonexistentDirectory(t *testing.T) { + ctx := context.Background() + nonexistentDir := "/nonexistent/directory" + + config := CleanupConfig{ + MaxAge: time.Hour, + ScanInterval: time.Hour, + DryRun: false, + } + + manager := NewCleanupManager(config) + + // This should not panic or error when directory doesn't exist + manager.cleanup(ctx, nonexistentDir) +} + +func TestShouldDeleteFile(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + + manager := NewCleanupManager(DefaultCleanupConfig()) + cutoff := time.Now().Add(-time.Hour) + + // Test with valid cache entry - old file + oldFile := filepath.Join(tempDir, "old.json") + oldEntry := cacheEntry{ + Data: json.RawMessage(`"data"`), + Timestamp: cutoff.Add(-time.Hour), // Before cutoff + } + oldData, err := json.Marshal(oldEntry) + require.NoError(t, err) + require.NoError(t, os.WriteFile(oldFile, oldData, 0o644)) + + shouldDelete, age := manager.shouldDeleteFile(ctx, oldFile, cutoff) + assert.True(t, shouldDelete, "Old file should be marked for deletion") + assert.Greater(t, age, 2*time.Hour, "Age should be calculated correctly") + + // Test with valid cache entry - recent file + recentFile := filepath.Join(tempDir, "recent.json") + recentEntry := cacheEntry{ + Data: json.RawMessage(`"data"`), + Timestamp: cutoff.Add(time.Hour), // After cutoff + } + recentData, err := json.Marshal(recentEntry) + require.NoError(t, err) + require.NoError(t, os.WriteFile(recentFile, recentData, 0o644)) + + shouldDelete, age = manager.shouldDeleteFile(ctx, recentFile, cutoff) + assert.False(t, shouldDelete, "Recent file should not be marked for deletion") + assert.Less(t, age, time.Hour, "Age should be calculated correctly") + + // Test with invalid JSON - should use file modification time + invalidFile := filepath.Join(tempDir, "invalid.json") + require.NoError(t, os.WriteFile(invalidFile, []byte("invalid"), 0o644)) + // Set modification time to be old + oldTime := cutoff.Add(-time.Hour) + require.NoError(t, os.Chtimes(invalidFile, oldTime, oldTime)) + + shouldDelete, age = manager.shouldDeleteFile(ctx, invalidFile, cutoff) + assert.True(t, shouldDelete, "Invalid file should be marked for deletion based on mod time") + assert.Greater(t, age, time.Hour, "Age should be based on modification time") +} + +func TestCleanupIntegrationWithFileCache(t *testing.T) { + tempDir := t.TempDir() + + // Create file cache which should start cleanup automatically + cache, err := newFileCacheWithBaseDir[string](tempDir) + require.NoError(t, err) + require.NotNil(t, cache.cleanupMgr) + + // Stop cleanup to prevent interference with test + cache.StopCleanup() + cache.cleanupMgr.Wait() + + // Verify cache directory was created + _, err = os.Stat(tempDir) + assert.False(t, os.IsNotExist(err), "Cache directory should exist") +} + +func TestCleanupManager_MultipleStartStop(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + + manager := NewCleanupManager(DefaultCleanupConfig()) + + // Start multiple times - should only start once + manager.Start(ctx, tempDir) + manager.Start(ctx, tempDir) // Second start should be ignored + assert.True(t, manager.running) + + // Stop multiple times - should be safe + manager.Stop() + manager.Stop() // Second stop should be safe + + manager.Wait() + assert.False(t, manager.running) +} + +func TestCleanupFileWalk(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + + config := CleanupConfig{ + MaxAge: time.Hour, + ScanInterval: time.Hour, + DryRun: false, + } + + manager := NewCleanupManager(config) + + // Create nested directory structure + subDir := filepath.Join(tempDir, "subdir") + require.NoError(t, os.MkdirAll(subDir, 0o755)) + + now := time.Now() + + // Create old files in both root and subdirectory + oldFile1 := filepath.Join(tempDir, "old1.json") + oldFile2 := filepath.Join(subDir, "old2.json") + + for _, file := range []string{oldFile1, oldFile2} { + oldEntry := cacheEntry{ + Data: json.RawMessage(`"old_data"`), + Timestamp: now.Add(-2 * time.Hour), + } + data, err := json.Marshal(oldEntry) + require.NoError(t, err) + require.NoError(t, os.WriteFile(file, data, 0o644)) + } + + // Run cleanup + manager.cleanup(ctx, tempDir) + + // Both files should be deleted + for _, file := range []string{oldFile1, oldFile2} { + _, err := os.Stat(file) + assert.True(t, os.IsNotExist(err), "File %s should be deleted", file) + } + + // Subdirectory should still exist + _, err := os.Stat(subDir) + assert.False(t, os.IsNotExist(err), "Subdirectory should still exist") +} diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 79d3c8568ec..a49c45edecb 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -17,10 +17,11 @@ import ( // FileCache implements the Cache interface using local disk storage. type FileCache[T any] struct { - baseDir string - mu sync.RWMutex - pending map[string]chan struct{} // Track pending writes - memCache map[string]T // In-memory cache for immediate access + baseDir string + mu sync.RWMutex + pending map[string]chan struct{} // Track pending writes + memCache map[string]T // In-memory cache for immediate access + cleanupMgr *CleanupManager // Background cleanup manager } // newFileCacheWithBaseDir creates a new file-based cache that stores data in the specified directory. @@ -29,11 +30,19 @@ func newFileCacheWithBaseDir[T any](baseDir string) (*FileCache[T], error) { return nil, fmt.Errorf("failed to create cache directory: %w", err) } - return &FileCache[T]{ - baseDir: baseDir, - pending: make(map[string]chan struct{}), - memCache: make(map[string]T), - }, nil + cleanupMgr := NewCleanupManager(DefaultCleanupConfig()) + + fc := &FileCache[T]{ + baseDir: baseDir, + pending: make(map[string]chan struct{}), + memCache: make(map[string]T), + cleanupMgr: cleanupMgr, + } + + // Start background cleanup (non-blocking) + cleanupMgr.Start(context.Background(), baseDir) + + return fc, nil } // NewFileCache creates a new file-based cache using UserCacheDir() + "databricks" + cached component name. @@ -234,3 +243,11 @@ func (fc *FileCache[T]) getCacheKey(fingerprint string) string { func (fc *FileCache[T]) getCachePath(cacheKey string) string { return filepath.Join(fc.baseDir, cacheKey+".json") } + +// StopCleanup stops the background cleanup process. +// This is non-blocking and will not wait for cleanup to complete. +func (fc *FileCache[T]) StopCleanup() { + if fc.cleanupMgr != nil { + fc.cleanupMgr.Stop() + } +} From d8aab3b3d91388af2ed273b9695590f02e86741c Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 23 Sep 2025 16:03:27 +0200 Subject: [PATCH 34/87] remove the ScanInterval setting --- libs/cache/cleanup.go | 32 +++++++++----------------------- libs/cache/cleanup_test.go | 38 +++++++++++++++----------------------- 2 files changed, 24 insertions(+), 46 deletions(-) diff --git a/libs/cache/cleanup.go b/libs/cache/cleanup.go index df9527b44af..72d35d2377c 100644 --- a/libs/cache/cleanup.go +++ b/libs/cache/cleanup.go @@ -14,17 +14,15 @@ import ( // CleanupConfig holds configuration for cache cleanup. type CleanupConfig struct { - MaxAge time.Duration // Maximum age of cache files before cleanup - ScanInterval time.Duration // How often to scan for old files - DryRun bool // If true, only logs what would be deleted + MaxAge time.Duration // Maximum age of cache files before cleanup + DryRun bool // If true, only logs what would be deleted } // DefaultCleanupConfig returns sensible defaults for cache cleanup. func DefaultCleanupConfig() CleanupConfig { return CleanupConfig{ - MaxAge: 7 * 24 * time.Hour, // 7 days - ScanInterval: 24 * time.Hour, // Daily cleanup - DryRun: false, + MaxAge: 7 * 24 * time.Hour, // 7 days + DryRun: false, } } @@ -47,7 +45,7 @@ func NewCleanupManager(config CleanupConfig) *CleanupManager { } } -// Start begins the background cleanup process. +// Start runs a one-time cleanup of cache files. // This is non-blocking and will not prevent the main process from exiting. func (cm *CleanupManager) Start(ctx context.Context, cacheDir string) { cm.mu.Lock() @@ -69,22 +67,10 @@ func (cm *CleanupManager) Start(ctx context.Context, cacheDir string) { log.Debugf(ctx, "[Cache Cleanup] Starting cleanup manager for directory: %s", cacheDir) - // Perform initial cleanup + // Perform one-time cleanup cm.cleanup(ctx, cacheDir) - // Set up periodic cleanup - ticker := time.NewTicker(cm.config.ScanInterval) - defer ticker.Stop() - - for { - select { - case <-cm.stopCh: - log.Debugf(ctx, "[Cache Cleanup] Cleanup manager stopped") - return - case <-ticker.C: - cm.cleanup(ctx, cacheDir) - } - } + log.Debugf(ctx, "[Cache Cleanup] Cleanup manager finished") }() } @@ -136,7 +122,7 @@ func (cm *CleanupManager) cleanup(ctx context.Context, cacheDir string) { scannedCount++ totalSize += info.Size() - shouldDelete, fileAge := cm.shouldDeleteFile(ctx, path, cutoff) + shouldDelete, fileAge := cm.shouldDeleteFile(path, cutoff) if shouldDelete { deletedSize += info.Size() deletedCount++ @@ -169,7 +155,7 @@ func (cm *CleanupManager) cleanup(ctx context.Context, cacheDir string) { } // shouldDeleteFile determines if a cache file should be deleted based on its age. -func (cm *CleanupManager) shouldDeleteFile(ctx context.Context, path string, cutoff time.Time) (bool, time.Duration) { +func (cm *CleanupManager) shouldDeleteFile(path string, cutoff time.Time) (bool, time.Duration) { // Try to read the cache entry to get the timestamp data, err := os.ReadFile(path) if err != nil { diff --git a/libs/cache/cleanup_test.go b/libs/cache/cleanup_test.go index 51244999de0..105cd65991a 100644 --- a/libs/cache/cleanup_test.go +++ b/libs/cache/cleanup_test.go @@ -15,7 +15,6 @@ import ( func TestDefaultCleanupConfig(t *testing.T) { config := DefaultCleanupConfig() assert.Equal(t, 7*24*time.Hour, config.MaxAge) - assert.Equal(t, 24*time.Hour, config.ScanInterval) assert.False(t, config.DryRun) } @@ -24,9 +23,8 @@ func TestCleanupManager_Start_Stop(t *testing.T) { tempDir := t.TempDir() config := CleanupConfig{ - MaxAge: time.Hour, - ScanInterval: 100 * time.Millisecond, - DryRun: false, + MaxAge: time.Hour, + DryRun: false, } manager := NewCleanupManager(config) @@ -58,9 +56,8 @@ func TestCleanupManager_CleanupOldFiles(t *testing.T) { tempDir := t.TempDir() config := CleanupConfig{ - MaxAge: time.Hour, - ScanInterval: time.Hour, // Long interval to prevent automatic cleanup during test - DryRun: false, + MaxAge: time.Hour, + DryRun: false, } manager := NewCleanupManager(config) @@ -111,9 +108,8 @@ func TestCleanupManager_DryRun(t *testing.T) { tempDir := t.TempDir() config := CleanupConfig{ - MaxAge: time.Hour, - ScanInterval: time.Hour, - DryRun: true, // Dry run mode + MaxAge: time.Hour, + DryRun: true, // Dry run mode } manager := NewCleanupManager(config) @@ -142,9 +138,8 @@ func TestCleanupManager_CorruptedFiles(t *testing.T) { tempDir := t.TempDir() config := CleanupConfig{ - MaxAge: time.Hour, - ScanInterval: time.Hour, - DryRun: false, + MaxAge: time.Hour, + DryRun: false, } manager := NewCleanupManager(config) @@ -180,9 +175,8 @@ func TestCleanupManager_NonexistentDirectory(t *testing.T) { nonexistentDir := "/nonexistent/directory" config := CleanupConfig{ - MaxAge: time.Hour, - ScanInterval: time.Hour, - DryRun: false, + MaxAge: time.Hour, + DryRun: false, } manager := NewCleanupManager(config) @@ -192,7 +186,6 @@ func TestCleanupManager_NonexistentDirectory(t *testing.T) { } func TestShouldDeleteFile(t *testing.T) { - ctx := context.Background() tempDir := t.TempDir() manager := NewCleanupManager(DefaultCleanupConfig()) @@ -208,7 +201,7 @@ func TestShouldDeleteFile(t *testing.T) { require.NoError(t, err) require.NoError(t, os.WriteFile(oldFile, oldData, 0o644)) - shouldDelete, age := manager.shouldDeleteFile(ctx, oldFile, cutoff) + shouldDelete, age := manager.shouldDeleteFile(oldFile, cutoff) assert.True(t, shouldDelete, "Old file should be marked for deletion") assert.Greater(t, age, 2*time.Hour, "Age should be calculated correctly") @@ -222,7 +215,7 @@ func TestShouldDeleteFile(t *testing.T) { require.NoError(t, err) require.NoError(t, os.WriteFile(recentFile, recentData, 0o644)) - shouldDelete, age = manager.shouldDeleteFile(ctx, recentFile, cutoff) + shouldDelete, age = manager.shouldDeleteFile(recentFile, cutoff) assert.False(t, shouldDelete, "Recent file should not be marked for deletion") assert.Less(t, age, time.Hour, "Age should be calculated correctly") @@ -233,7 +226,7 @@ func TestShouldDeleteFile(t *testing.T) { oldTime := cutoff.Add(-time.Hour) require.NoError(t, os.Chtimes(invalidFile, oldTime, oldTime)) - shouldDelete, age = manager.shouldDeleteFile(ctx, invalidFile, cutoff) + shouldDelete, age = manager.shouldDeleteFile(invalidFile, cutoff) assert.True(t, shouldDelete, "Invalid file should be marked for deletion based on mod time") assert.Greater(t, age, time.Hour, "Age should be based on modification time") } @@ -279,9 +272,8 @@ func TestCleanupFileWalk(t *testing.T) { tempDir := t.TempDir() config := CleanupConfig{ - MaxAge: time.Hour, - ScanInterval: time.Hour, - DryRun: false, + MaxAge: time.Hour, + DryRun: false, } manager := NewCleanupManager(config) From 7519530409f404b04a462f245300ce3678d3d743 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Tue, 23 Sep 2025 16:32:00 +0200 Subject: [PATCH 35/87] store expiration time instead of timestamp in a cache entry --- .../config/mutator/populate_current_user.go | 2 +- libs/cache/cleanup.go | 36 ++++++-- libs/cache/cleanup_test.go | 63 ++++++++----- libs/cache/expiry_test.go | 91 +++++++++++++++++++ libs/cache/file_cache.go | 33 ++++--- libs/cache/file_cache_test.go | 16 ++-- 6 files changed, 184 insertions(+), 57 deletions(-) create mode 100644 libs/cache/expiry_test.go diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index 4207fed9029..73a17d3c893 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -39,7 +39,7 @@ func (m *populateCurrentUser) initializeCache(ctx context.Context) { } var err error - m.cache, err = cache.NewFileCache[*iam.User]("auth") + m.cache, err = cache.NewFileCache[*iam.User]("auth", 30) if err != nil { log.Debugf(ctx, "[Local Cache] Failed to initialize cache dir: %v\n", err) } diff --git a/libs/cache/cleanup.go b/libs/cache/cleanup.go index 72d35d2377c..d0dcede0499 100644 --- a/libs/cache/cleanup.go +++ b/libs/cache/cleanup.go @@ -106,7 +106,7 @@ func (cm *CleanupManager) cleanup(ctx context.Context, cacheDir string) { var deletedCount, scannedCount int var totalSize, deletedSize int64 - cutoff := time.Now().Add(-cm.config.MaxAge) + now := time.Now() err := filepath.Walk(cacheDir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -122,7 +122,7 @@ func (cm *CleanupManager) cleanup(ctx context.Context, cacheDir string) { scannedCount++ totalSize += info.Size() - shouldDelete, fileAge := cm.shouldDeleteFile(path, cutoff) + shouldDelete, fileAge := cm.shouldDeleteFile(path, now) if shouldDelete { deletedSize += info.Size() deletedCount++ @@ -154,15 +154,16 @@ func (cm *CleanupManager) cleanup(ctx context.Context, cacheDir string) { action, deletedCount, float64(deletedSize)/(1024*1024)) } -// shouldDeleteFile determines if a cache file should be deleted based on its age. -func (cm *CleanupManager) shouldDeleteFile(path string, cutoff time.Time) (bool, time.Duration) { - // Try to read the cache entry to get the timestamp +// shouldDeleteFile determines if a cache file should be deleted based on its expiry. +func (cm *CleanupManager) shouldDeleteFile(path string, now time.Time) (bool, time.Duration) { + // Try to read the cache entry to get the expiry data, err := os.ReadFile(path) if err != nil { // If we can't read the file, use file modification time as fallback if info, statErr := os.Stat(path); statErr == nil { age := time.Since(info.ModTime()) - return info.ModTime().Before(cutoff), age + // Use MaxAge fallback for files without expiry information + return info.ModTime().Add(cm.config.MaxAge).Before(now), age } return true, time.Duration(0) // Delete unreadable files } @@ -172,11 +173,28 @@ func (cm *CleanupManager) shouldDeleteFile(path string, cutoff time.Time) (bool, // If we can't parse the cache entry, use file modification time as fallback if info, statErr := os.Stat(path); statErr == nil { age := time.Since(info.ModTime()) - return info.ModTime().Before(cutoff), age + // Use MaxAge fallback for files without expiry information + return info.ModTime().Add(cm.config.MaxAge).Before(now), age } return true, time.Duration(0) // Delete unparseable files } - age := time.Since(entry.Timestamp) - return entry.Timestamp.Before(cutoff), age + // Check if the file has expired + if !entry.Expiry.IsZero() { + isExpired := entry.Expiry.Before(now) + age := now.Sub(entry.Expiry) + if age < 0 { + age = 0 // File hasn't expired yet + } + return isExpired, age + } + + // Fallback to Timestamp field for backward compatibility + if !entry.Timestamp.IsZero() { + age := time.Since(entry.Timestamp) + return entry.Timestamp.Add(cm.config.MaxAge).Before(now), age + } + + // If neither expiry nor timestamp is available, delete the file + return true, time.Duration(0) } diff --git a/libs/cache/cleanup_test.go b/libs/cache/cleanup_test.go index 105cd65991a..91e1df1c58d 100644 --- a/libs/cache/cleanup_test.go +++ b/libs/cache/cleanup_test.go @@ -189,53 +189,68 @@ func TestShouldDeleteFile(t *testing.T) { tempDir := t.TempDir() manager := NewCleanupManager(DefaultCleanupConfig()) - cutoff := time.Now().Add(-time.Hour) + now := time.Now() - // Test with valid cache entry - old file - oldFile := filepath.Join(tempDir, "old.json") - oldEntry := cacheEntry{ - Data: json.RawMessage(`"data"`), - Timestamp: cutoff.Add(-time.Hour), // Before cutoff + // Test with valid cache entry with expiry - expired file + expiredFile := filepath.Join(tempDir, "expired.json") + expiredEntry := cacheEntry{ + Data: json.RawMessage(`"data"`), + Expiry: now.Add(-time.Hour), // Expired 1 hour ago } - oldData, err := json.Marshal(oldEntry) + expiredData, err := json.Marshal(expiredEntry) require.NoError(t, err) - require.NoError(t, os.WriteFile(oldFile, oldData, 0o644)) + require.NoError(t, os.WriteFile(expiredFile, expiredData, 0o644)) - shouldDelete, age := manager.shouldDeleteFile(oldFile, cutoff) - assert.True(t, shouldDelete, "Old file should be marked for deletion") - assert.Greater(t, age, 2*time.Hour, "Age should be calculated correctly") + shouldDelete, age := manager.shouldDeleteFile(expiredFile, now) + assert.True(t, shouldDelete, "Expired file should be marked for deletion") + assert.GreaterOrEqual(t, age, time.Hour, "Age should reflect time since expiry") - // Test with valid cache entry - recent file - recentFile := filepath.Join(tempDir, "recent.json") - recentEntry := cacheEntry{ + // Test with valid cache entry with expiry - not expired file + validFile := filepath.Join(tempDir, "valid.json") + validEntry := cacheEntry{ + Data: json.RawMessage(`"data"`), + Expiry: now.Add(time.Hour), // Expires in 1 hour + } + validData, err := json.Marshal(validEntry) + require.NoError(t, err) + require.NoError(t, os.WriteFile(validFile, validData, 0o644)) + + shouldDelete, age = manager.shouldDeleteFile(validFile, now) + assert.False(t, shouldDelete, "Valid file should not be marked for deletion") + assert.Equal(t, time.Duration(0), age, "Age should be 0 for unexpired files") + + // Test with legacy timestamp field (backward compatibility) + legacyFile := filepath.Join(tempDir, "legacy.json") + legacyEntry := cacheEntry{ Data: json.RawMessage(`"data"`), - Timestamp: cutoff.Add(time.Hour), // After cutoff + Timestamp: now.Add(-2 * time.Hour), // Created 2 hours ago } - recentData, err := json.Marshal(recentEntry) + legacyData, err := json.Marshal(legacyEntry) require.NoError(t, err) - require.NoError(t, os.WriteFile(recentFile, recentData, 0o644)) + require.NoError(t, os.WriteFile(legacyFile, legacyData, 0o644)) - shouldDelete, age = manager.shouldDeleteFile(recentFile, cutoff) - assert.False(t, shouldDelete, "Recent file should not be marked for deletion") - assert.Less(t, age, time.Hour, "Age should be calculated correctly") + shouldDelete, age = manager.shouldDeleteFile(legacyFile, now) + // Should not be deleted since MaxAge is 7 days by default, but 2 hours < 7 days + assert.False(t, shouldDelete, "Legacy file should not be deleted if within MaxAge") + assert.Greater(t, age, 2*time.Hour, "Age should be based on timestamp") // Test with invalid JSON - should use file modification time invalidFile := filepath.Join(tempDir, "invalid.json") require.NoError(t, os.WriteFile(invalidFile, []byte("invalid"), 0o644)) // Set modification time to be old - oldTime := cutoff.Add(-time.Hour) + oldTime := now.Add(-8 * 24 * time.Hour) // 8 days ago (beyond default MaxAge) require.NoError(t, os.Chtimes(invalidFile, oldTime, oldTime)) - shouldDelete, age = manager.shouldDeleteFile(invalidFile, cutoff) + shouldDelete, age = manager.shouldDeleteFile(invalidFile, now) assert.True(t, shouldDelete, "Invalid file should be marked for deletion based on mod time") - assert.Greater(t, age, time.Hour, "Age should be based on modification time") + assert.Greater(t, age, 7*24*time.Hour, "Age should be based on modification time") } func TestCleanupIntegrationWithFileCache(t *testing.T) { tempDir := t.TempDir() // Create file cache which should start cleanup automatically - cache, err := newFileCacheWithBaseDir[string](tempDir) + cache, err := newFileCacheWithBaseDir[string](tempDir, 60) // 1 hour for tests require.NoError(t, err) require.NotNil(t, cache.cleanupMgr) diff --git a/libs/cache/expiry_test.go b/libs/cache/expiry_test.go new file mode 100644 index 00000000000..0d85b6277a9 --- /dev/null +++ b/libs/cache/expiry_test.go @@ -0,0 +1,91 @@ +package cache + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestFileCacheExpiryBehavior tests that the new expiry-based cache works as expected +func TestFileCacheExpiryBehavior(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + + // Create cache with 1 minute expiry + cache, err := newFileCacheWithBaseDir[string](tempDir, 1) + require.NoError(t, err) + defer cache.StopCleanup() + + fingerprint := struct { + Key string `json:"key"` + }{ + Key: "test-expiry", + } + + // Compute and store a value + result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + return "test-value", nil + }) + require.NoError(t, err) + assert.Equal(t, "test-value", result) + + // Allow time for async write to complete + time.Sleep(100 * time.Millisecond) + + // Find the cache file and verify it has the correct expiry + cacheFiles, err := filepath.Glob(filepath.Join(tempDir, "*.json")) + require.NoError(t, err) + require.Len(t, cacheFiles, 1) + + // Read the cache file and check expiry + data, err := os.ReadFile(cacheFiles[0]) + require.NoError(t, err) + + var entry cacheEntry + err = json.Unmarshal(data, &entry) + require.NoError(t, err) + + // Verify expiry is set and is approximately 1 minute from now + assert.False(t, entry.Expiry.IsZero(), "Expiry should be set") + expectedExpiry := time.Now().Add(time.Minute) + timeDiff := entry.Expiry.Sub(expectedExpiry).Abs() + assert.Less(t, timeDiff, 10*time.Second, "Expiry should be approximately 1 minute from creation time") + + // Verify cleanup would identify an expired file + manager := NewCleanupManager(DefaultCleanupConfig()) + futureTime := time.Now().Add(2 * time.Minute) // 2 minutes from now, past expiry + shouldDelete, age := manager.shouldDeleteFile(cacheFiles[0], futureTime) + assert.True(t, shouldDelete, "File should be marked for deletion when past expiry") + assert.Greater(t, age, time.Duration(0), "Age should be positive when expired") +} + +// TestLegacyTimestampCompatibility tests that old cache files with timestamp still work +func TestLegacyTimestampCompatibility(t *testing.T) { + tempDir := t.TempDir() + + // Create a legacy cache file with timestamp + legacyEntry := cacheEntry{ + Data: json.RawMessage(`"legacy-value"`), + Timestamp: time.Now().Add(-time.Hour), // 1 hour ago + } + legacyData, err := json.Marshal(legacyEntry) + require.NoError(t, err) + + legacyFile := filepath.Join(tempDir, "legacy.json") + require.NoError(t, os.WriteFile(legacyFile, legacyData, 0o644)) + + // Test cleanup logic handles legacy files correctly + manager := NewCleanupManager(DefaultCleanupConfig()) + now := time.Now() + + // Should not delete a 1-hour-old file (default MaxAge is 7 days) + shouldDelete, age := manager.shouldDeleteFile(legacyFile, now) + assert.False(t, shouldDelete, "Legacy file should not be deleted if within MaxAge") + assert.Greater(t, age, time.Hour, "Age should be calculated from timestamp") +} diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index a49c45edecb..6b38503e676 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -17,15 +17,16 @@ import ( // FileCache implements the Cache interface using local disk storage. type FileCache[T any] struct { - baseDir string - mu sync.RWMutex - pending map[string]chan struct{} // Track pending writes - memCache map[string]T // In-memory cache for immediate access - cleanupMgr *CleanupManager // Background cleanup manager + baseDir string + expiryMinutes int + mu sync.RWMutex + pending map[string]chan struct{} // Track pending writes + memCache map[string]T // In-memory cache for immediate access + cleanupMgr *CleanupManager // Background cleanup manager } // newFileCacheWithBaseDir creates a new file-based cache that stores data in the specified directory. -func newFileCacheWithBaseDir[T any](baseDir string) (*FileCache[T], error) { +func newFileCacheWithBaseDir[T any](baseDir string, expiryMinutes int) (*FileCache[T], error) { if err := os.MkdirAll(baseDir, 0o755); err != nil { return nil, fmt.Errorf("failed to create cache directory: %w", err) } @@ -33,10 +34,11 @@ func newFileCacheWithBaseDir[T any](baseDir string) (*FileCache[T], error) { cleanupMgr := NewCleanupManager(DefaultCleanupConfig()) fc := &FileCache[T]{ - baseDir: baseDir, - pending: make(map[string]chan struct{}), - memCache: make(map[string]T), - cleanupMgr: cleanupMgr, + baseDir: baseDir, + expiryMinutes: expiryMinutes, + pending: make(map[string]chan struct{}), + memCache: make(map[string]T), + cleanupMgr: cleanupMgr, } // Start background cleanup (non-blocking) @@ -46,20 +48,21 @@ func newFileCacheWithBaseDir[T any](baseDir string) (*FileCache[T], error) { } // NewFileCache creates a new file-based cache using UserCacheDir() + "databricks" + cached component name. -func NewFileCache[T any](component string) (*FileCache[T], error) { +func NewFileCache[T any](component string, expiryMinutes int) (*FileCache[T], error) { userCacheDir, err := os.UserCacheDir() if err != nil { return nil, fmt.Errorf("failed to get user cache directory: %w", err) } baseDir := filepath.Join(userCacheDir, "databricks", component) - return newFileCacheWithBaseDir[T](baseDir) + return newFileCacheWithBaseDir[T](baseDir, expiryMinutes) } // cacheEntry represents the structure of a cached item on disk. type cacheEntry struct { Data json.RawMessage `json:"data"` - Timestamp time.Time `json:"timestamp"` + Expiry time.Time `json:"expiry"` + Timestamp time.Time `json:"timestamp,omitempty"` // For backward compatibility } // GetOrCompute retrieves cached content or computes it using the provided function. @@ -195,8 +198,8 @@ func (fc *FileCache[T]) writeToCache(cachePath string, data any) { } entry := cacheEntry{ - Data: serializedData, - Timestamp: time.Now(), + Data: serializedData, + Expiry: time.Now().Add(time.Duration(fc.expiryMinutes) * time.Minute), } entryData, err := json.Marshal(entry) diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index b6dad9eb04d..8042e33a2ed 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -17,7 +17,7 @@ func TestNewFileCache(t *testing.T) { tempDir := t.TempDir() cacheDir := filepath.Join(tempDir, "cache") - cache, err := newFileCacheWithBaseDir[string](cacheDir) + cache, err := newFileCacheWithBaseDir[string](cacheDir, 60) // 1 hour for tests require.NoError(t, err) assert.NotNil(t, cache) assert.Equal(t, cacheDir, cache.baseDir) @@ -50,7 +50,7 @@ func TestNewFileCacheWithExistingDirectory(t *testing.T) { err := os.MkdirAll(cacheDir, 0o700) require.NoError(t, err) - cache, err := newFileCacheWithBaseDir[string](cacheDir) + cache, err := newFileCacheWithBaseDir[string](cacheDir, 60) // 1 hour for tests require.NoError(t, err) assert.NotNil(t, cache) assert.Equal(t, cacheDir, cache.baseDir) @@ -60,7 +60,7 @@ func TestNewFileCacheInvalidPath(t *testing.T) { // Try to create cache in a location that should fail invalidPath := "/root/invalid/path/that/should/not/exist" - cache, err := newFileCacheWithBaseDir[string](invalidPath) + cache, err := newFileCacheWithBaseDir[string](invalidPath, 60) // 1 hour for tests if err != nil { assert.Nil(t, cache) assert.Contains(t, err.Error(), "failed to create cache directory") @@ -70,7 +70,7 @@ func TestNewFileCacheInvalidPath(t *testing.T) { func TestFileCacheGetOrCompute(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](tempDir) + cache, err := newFileCacheWithBaseDir[string](tempDir, 60) // 1 hour for tests require.NoError(t, err) fingerprint := struct { @@ -110,7 +110,7 @@ func TestFileCacheGetOrCompute(t *testing.T) { func TestFileCacheGetOrComputeError(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](tempDir) + cache, err := newFileCacheWithBaseDir[string](tempDir, 60) // 1 hour for tests require.NoError(t, err) fingerprint := struct { @@ -132,7 +132,7 @@ func TestFileCacheGetOrComputeError(t *testing.T) { func TestFileCacheGetOrComputeConcurrency(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](tempDir) + cache, err := newFileCacheWithBaseDir[string](tempDir, 60) // 1 hour for tests require.NoError(t, err) fingerprint := struct { @@ -177,7 +177,7 @@ func TestFileCacheGetOrComputeConcurrency(t *testing.T) { func TestFileCacheGetOrComputeContextCancellation(t *testing.T) { tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](tempDir) + cache, err := newFileCacheWithBaseDir[string](tempDir, 60) // 1 hour for tests require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -200,7 +200,7 @@ func TestFileCacheGetOrComputeContextCancellation(t *testing.T) { func TestFingerprintDeterministic(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](tempDir) + cache, err := newFileCacheWithBaseDir[string](tempDir, 60) // 1 hour for tests require.NoError(t, err) // Create two identical structs with fields in different JSON order From 11e8702932cb350c2b0d815dd7f85da21704e588 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Wed, 24 Sep 2025 14:14:58 +0200 Subject: [PATCH 36/87] rename file_cache_expiry_test.go; change the assertion --- libs/cache/{expiry_test.go => file_cache_expiry_test.go} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename libs/cache/{expiry_test.go => file_cache_expiry_test.go} (94%) diff --git a/libs/cache/expiry_test.go b/libs/cache/file_cache_expiry_test.go similarity index 94% rename from libs/cache/expiry_test.go rename to libs/cache/file_cache_expiry_test.go index 0d85b6277a9..ab2fbb32080 100644 --- a/libs/cache/expiry_test.go +++ b/libs/cache/file_cache_expiry_test.go @@ -62,7 +62,7 @@ func TestFileCacheExpiryBehavior(t *testing.T) { futureTime := time.Now().Add(2 * time.Minute) // 2 minutes from now, past expiry shouldDelete, age := manager.shouldDeleteFile(cacheFiles[0], futureTime) assert.True(t, shouldDelete, "File should be marked for deletion when past expiry") - assert.Greater(t, age, time.Duration(0), "Age should be positive when expired") + assert.GreaterOrEqual(t, age, time.Duration(0), "Age should be positive when expired") } // TestLegacyTimestampCompatibility tests that old cache files with timestamp still work @@ -87,5 +87,5 @@ func TestLegacyTimestampCompatibility(t *testing.T) { // Should not delete a 1-hour-old file (default MaxAge is 7 days) shouldDelete, age := manager.shouldDeleteFile(legacyFile, now) assert.False(t, shouldDelete, "Legacy file should not be deleted if within MaxAge") - assert.Greater(t, age, time.Hour, "Age should be calculated from timestamp") + assert.GreaterOrEqual(t, age, time.Hour, "Age should be calculated from timestamp") } From c6a1da74e6f773b19217b18d7fb63c3bbbd74354 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Wed, 24 Sep 2025 15:35:14 +0200 Subject: [PATCH 37/87] add telemetry on caching events --- acceptance/cache/exploratory/output.txt | 11 ++++++++ acceptance/cache/exploratory/script | 6 +++++ acceptance/cache/exploratory/test.toml | 2 ++ .../config/mutator/populate_current_user.go | 6 ++--- libs/cache/file_cache.go | 27 +++++++++++++++++-- 5 files changed, 47 insertions(+), 5 deletions(-) diff --git a/acceptance/cache/exploratory/output.txt b/acceptance/cache/exploratory/output.txt index b2bec4c8d85..9fda4f8ddaf 100644 --- a/acceptance/cache/exploratory/output.txt +++ b/acceptance/cache/exploratory/output.txt @@ -12,3 +12,14 @@ [DEBUG_TIMESTAMP] Debug: [Local Cache] using fingerprint with hash: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit: disk-read + +=== Bundle deploy should send telemetry values + +>>> [CLI] bundle deploy -p dogfood +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/exploratory-cache-test/default/files... +Deploying resources... +Deployment complete! + +>>> print_telemetry_bool_values +local.cache.attempt true +local.cache.hit true diff --git a/acceptance/cache/exploratory/script b/acceptance/cache/exploratory/script index 9c225d79684..5646a4f1263 100644 --- a/acceptance/cache/exploratory/script +++ b/acceptance/cache/exploratory/script @@ -5,3 +5,9 @@ trace $CLI bundle validate -p dogfood --debug 2>&1 | grep "Local Cache" | grep - title "Second call in a session is expected to be a cache hit\n" trace $CLI bundle validate -p dogfood --debug 2>&1 | grep "Local Cache" | grep -v "cache path" + +title "Bundle deploy should send telemetry values\n" +trace $CLI bundle deploy -p dogfood + +trace print_telemetry_bool_values | grep "local.cache" +rm out.requests.txt diff --git a/acceptance/cache/exploratory/test.toml b/acceptance/cache/exploratory/test.toml index 2853ed8cf72..f931adddecc 100644 --- a/acceptance/cache/exploratory/test.toml +++ b/acceptance/cache/exploratory/test.toml @@ -1,6 +1,8 @@ Cloud=false Local=true +RecordRequests = true + [[Repls]] Old = '\d\d:\d\d:\d\d' New = "[DEBUG_TIMESTAMP]" diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index 73a17d3c893..dd634206781 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -28,7 +28,7 @@ func PopulateCurrentUser() bundle.Mutator { } // initializeCache sets up the cache for authorization headers if not already initialized -func (m *populateCurrentUser) initializeCache(ctx context.Context) { +func (m *populateCurrentUser) initializeCache(ctx context.Context, b *bundle.Bundle) { if m.cache != nil { return } @@ -39,7 +39,7 @@ func (m *populateCurrentUser) initializeCache(ctx context.Context) { } var err error - m.cache, err = cache.NewFileCache[*iam.User]("auth", 30) + m.cache, err = cache.NewFileCache[*iam.User]("auth", 30, &b.Metrics) if err != nil { log.Debugf(ctx, "[Local Cache] Failed to initialize cache dir: %v\n", err) } @@ -53,7 +53,7 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. if b.Config.Workspace.CurrentUser != nil { return nil } - m.initializeCache(ctx) + m.initializeCache(ctx, b) w := b.WorkspaceClient() fingerprint := struct { diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 6b38503e676..4fbd4065fa4 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -12,6 +12,8 @@ import ( "sync" "time" + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/log" ) @@ -23,6 +25,7 @@ type FileCache[T any] struct { pending map[string]chan struct{} // Track pending writes memCache map[string]T // In-memory cache for immediate access cleanupMgr *CleanupManager // Background cleanup manager + metrics *bundle.Metrics // Telemetry metrics } // newFileCacheWithBaseDir creates a new file-based cache that stores data in the specified directory. @@ -48,14 +51,19 @@ func newFileCacheWithBaseDir[T any](baseDir string, expiryMinutes int) (*FileCac } // NewFileCache creates a new file-based cache using UserCacheDir() + "databricks" + cached component name. -func NewFileCache[T any](component string, expiryMinutes int) (*FileCache[T], error) { +func NewFileCache[T any](component string, expiryMinutes int, metrics *bundle.Metrics) (*FileCache[T], error) { userCacheDir, err := os.UserCacheDir() if err != nil { return nil, fmt.Errorf("failed to get user cache directory: %w", err) } baseDir := filepath.Join(userCacheDir, "databricks", component) - return newFileCacheWithBaseDir[T](baseDir, expiryMinutes) + fc, err := newFileCacheWithBaseDir[T](baseDir, expiryMinutes) + if err != nil { + return nil, err + } + fc.metrics = metrics + return fc, nil } // cacheEntry represents the structure of a cached item on disk. @@ -65,6 +73,12 @@ type cacheEntry struct { Timestamp time.Time `json:"timestamp,omitempty"` // For backward compatibility } +func (fc *FileCache[T]) addTelemetryMetric(key string) { + if fc.metrics != nil { + fc.metrics.SetBoolValue(key, true) + } +} + // GetOrCompute retrieves cached content or computes it using the provided function. func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compute func(ctx context.Context) (T, error)) (T, error) { var zero T @@ -73,6 +87,8 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu fingerprintHash, err := fingerprintToHash(fingerprint) log.Debugf(ctx, "[Local Cache] using fingerprint with hash: %s\n", fingerprintHash) + fc.addTelemetryMetric("local.cache.attempt") + if err != nil { log.Debugf(ctx, "[Local Cache] cache miss: non-compliant fingerprint\n") return zero, fmt.Errorf("failed to convert fingerprint to string: %w", err) @@ -89,6 +105,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu if data, found := fc.memCache[cacheKey]; found { fc.mu.RUnlock() log.Debugf(ctx, "[Local Cache] cache hit: in-memory\n") + fc.addTelemetryMetric("local.cache.hit") return data, nil } fc.mu.RUnlock() @@ -100,6 +117,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu fc.memCache[cacheKey] = data fc.mu.Unlock() log.Debugf(ctx, "[Local Cache] cache hit: disk-read\n") + fc.addTelemetryMetric("local.cache.hit") return data, nil } @@ -115,11 +133,13 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu if data, found := fc.memCache[cacheKey]; found { fc.mu.RUnlock() log.Debugf(ctx, "[Local Cache] cache hit: in-memory from pending write\n") + fc.addTelemetryMetric("local.cache.hit") return data, nil } fc.mu.RUnlock() case <-ctx.Done(): log.Debugf(ctx, "[Local Cache] cache miss: no hit while waiting for pending write\n") + fc.addTelemetryMetric("local.cache.miss") return zero, ctx.Err() } } else { @@ -140,6 +160,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu select { case <-ctx.Done(): log.Debugf(ctx, "[Local Cache] cache miss: context is already cancelled\n") + fc.addTelemetryMetric("local.cache.miss") return zero, ctx.Err() default: } @@ -148,6 +169,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu result, err := compute(ctx) if err != nil { log.Debugf(ctx, "[Local Cache] error while caching: %v\n", err) + fc.addTelemetryMetric("local.cache.error") return zero, err } @@ -161,6 +183,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu go fc.writeToCache(cachePath, result) log.Debugf(ctx, "[Local Cache] cache miss, but stored the compute result for future calls\n") + fc.addTelemetryMetric("local.cache.miss") return result, nil } From 4bbc589e29721d181412e47b37b6548495c53b73 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Wed, 24 Sep 2025 16:34:41 +0200 Subject: [PATCH 38/87] add new `cache clear` command --- acceptance/cache/clear/databricks.yml | 2 ++ acceptance/cache/clear/out.test.toml | 5 +++++ acceptance/cache/clear/output.txt | 24 +++++++++++++++++++++ acceptance/cache/clear/script | 12 +++++++++++ acceptance/cache/clear/test.toml | 10 +++++++++ cmd/cache/cache.go | 30 +++++++++++++++++++++++++++ cmd/cmd.go | 2 ++ libs/cache/file_cache.go | 14 ++++++++++--- libs/cache/file_cache_clear.go | 30 +++++++++++++++++++++++++++ 9 files changed, 126 insertions(+), 3 deletions(-) create mode 100644 acceptance/cache/clear/databricks.yml create mode 100644 acceptance/cache/clear/out.test.toml create mode 100644 acceptance/cache/clear/output.txt create mode 100644 acceptance/cache/clear/script create mode 100644 acceptance/cache/clear/test.toml create mode 100644 cmd/cache/cache.go create mode 100644 libs/cache/file_cache_clear.go diff --git a/acceptance/cache/clear/databricks.yml b/acceptance/cache/clear/databricks.yml new file mode 100644 index 00000000000..5557aa4ad43 --- /dev/null +++ b/acceptance/cache/clear/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: cache-clear-test diff --git a/acceptance/cache/clear/out.test.toml b/acceptance/cache/clear/out.test.toml new file mode 100644 index 00000000000..e092fd5ed6a --- /dev/null +++ b/acceptance/cache/clear/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct-exp"] diff --git a/acceptance/cache/clear/output.txt b/acceptance/cache/clear/output.txt new file mode 100644 index 00000000000..50239eabf68 --- /dev/null +++ b/acceptance/cache/clear/output.txt @@ -0,0 +1,24 @@ + +=== First call in a session is expected to be a cache miss: +[DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 +[DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled +[DEBUG_TIMESTAMP] Debug: [Local Cache] using fingerprint with hash: [SHA256_HASH] +[DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, but stored the compute result for future calls + +=== Second call in a session is expected to be a cache hit +[DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 +[DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled +[DEBUG_TIMESTAMP] Debug: [Local Cache] using fingerprint with hash: [SHA256_HASH] +[DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit: disk-read + +>>> [CLI] cache clear +Cache cleared successfully + +=== First call after a clear is expected to be a cache miss: +[DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 +[DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled +[DEBUG_TIMESTAMP] Debug: [Local Cache] using fingerprint with hash: [SHA256_HASH] +[DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, but stored the compute result for future calls diff --git a/acceptance/cache/clear/script b/acceptance/cache/clear/script new file mode 100644 index 00000000000..11863ee18b1 --- /dev/null +++ b/acceptance/cache/clear/script @@ -0,0 +1,12 @@ +export DATABRICKS_EXPERIMENTAL_CACHE_ENABLED=true + +title "First call in a session is expected to be a cache miss:\n" +trace $CLI bundle validate --debug 2>&1 | grep "Local Cache" | grep -v "cache path" + +title "Second call in a session is expected to be a cache hit\n" +trace $CLI bundle validate --debug 2>&1 | grep "Local Cache" | grep -v "cache path" + +trace $CLI cache clear + +title "First call after a clear is expected to be a cache miss:\n" +trace $CLI bundle validate --debug 2>&1 | grep "Local Cache" | grep -v "cache path" diff --git a/acceptance/cache/clear/test.toml b/acceptance/cache/clear/test.toml new file mode 100644 index 00000000000..2853ed8cf72 --- /dev/null +++ b/acceptance/cache/clear/test.toml @@ -0,0 +1,10 @@ +Cloud=false +Local=true + +[[Repls]] +Old = '\d\d:\d\d:\d\d' +New = "[DEBUG_TIMESTAMP]" + +[[Repls]] +Old = '[a-f0-9]{64}' +New = "[SHA256_HASH]" diff --git a/cmd/cache/cache.go b/cmd/cache/cache.go new file mode 100644 index 00000000000..18c213ff323 --- /dev/null +++ b/cmd/cache/cache.go @@ -0,0 +1,30 @@ +package cache + +import ( + "github.com/databricks/cli/libs/cache" + "github.com/spf13/cobra" +) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "cache", + Short: "Local cache related commands", + Long: "Manage local cache used by the Databricks CLI for improved performance", + } + + cmd.AddCommand(newClearCommand()) + return cmd +} + +func newClearCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "clear", + Short: "Clear all local cache files", + Long: "Remove all cached files stored locally by the Databricks CLI", + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + return cache.ClearFileCache(ctx) + }, + } + return cmd +} diff --git a/cmd/cmd.go b/cmd/cmd.go index 3e7925e7eda..3051aed5584 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/cmd/api" "github.com/databricks/cli/cmd/auth" "github.com/databricks/cli/cmd/bundle" + "github.com/databricks/cli/cmd/cache" "github.com/databricks/cli/cmd/configure" "github.com/databricks/cli/cmd/fs" "github.com/databricks/cli/cmd/labs" @@ -70,6 +71,7 @@ func New(ctx context.Context) *cobra.Command { cli.AddCommand(api.New()) cli.AddCommand(auth.New()) cli.AddCommand(bundle.New()) + cli.AddCommand(cache.New()) cli.AddCommand(psql.New()) cli.AddCommand(configure.New()) cli.AddCommand(fs.New()) diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 4fbd4065fa4..ff30e66b908 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -50,14 +50,22 @@ func newFileCacheWithBaseDir[T any](baseDir string, expiryMinutes int) (*FileCac return fc, nil } +func getCacheBaseDir() (string, error) { + userCacheDir, err := os.UserCacheDir() + if err != nil { + return "", fmt.Errorf("failed to get user cache directory: %w", err) + } + return filepath.Join(userCacheDir, "databricks"), nil +} + // NewFileCache creates a new file-based cache using UserCacheDir() + "databricks" + cached component name. func NewFileCache[T any](component string, expiryMinutes int, metrics *bundle.Metrics) (*FileCache[T], error) { - userCacheDir, err := os.UserCacheDir() + cacheBaseDir, err := getCacheBaseDir() if err != nil { - return nil, fmt.Errorf("failed to get user cache directory: %w", err) + return nil, err } - baseDir := filepath.Join(userCacheDir, "databricks", component) + baseDir := filepath.Join(cacheBaseDir, component) fc, err := newFileCacheWithBaseDir[T](baseDir, expiryMinutes) if err != nil { return nil, err diff --git a/libs/cache/file_cache_clear.go b/libs/cache/file_cache_clear.go new file mode 100644 index 00000000000..28a43ee3888 --- /dev/null +++ b/libs/cache/file_cache_clear.go @@ -0,0 +1,30 @@ +package cache + +import ( + "context" + "os" + + "github.com/databricks/cli/libs/cmdio" +) + +func ClearFileCache(ctx context.Context) error { + databricksCacheDir, err := getCacheBaseDir() + if err != nil { + return err + } + + // Check if the cache directory exists + if _, err := os.Stat(databricksCacheDir); os.IsNotExist(err) { + cmdio.LogString(ctx, "No cache directory found, nothing to clear") + return nil + } + + // Remove the entire databricks cache directory + err = os.RemoveAll(databricksCacheDir) + if err != nil { + return err + } + + cmdio.LogString(ctx, "Cache cleared successfully") + return nil +} From bcc8dc97b0c068715c290bdca3c483e60a875424 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Wed, 24 Sep 2025 17:29:50 +0200 Subject: [PATCH 39/87] fix tests --- acceptance/cache/clear/script | 2 ++ acceptance/help/output.txt | 1 + 2 files changed, 3 insertions(+) diff --git a/acceptance/cache/clear/script b/acceptance/cache/clear/script index 11863ee18b1..ada456d9513 100644 --- a/acceptance/cache/clear/script +++ b/acceptance/cache/clear/script @@ -1,5 +1,7 @@ export DATABRICKS_EXPERIMENTAL_CACHE_ENABLED=true +$CLI cache clear + title "First call in a session is expected to be a cache miss:\n" trace $CLI bundle validate --debug 2>&1 | grep "Local Cache" | grep -v "cache path" diff --git a/acceptance/help/output.txt b/acceptance/help/output.txt index 6c454de344a..869a3cc1c6b 100644 --- a/acceptance/help/output.txt +++ b/acceptance/help/output.txt @@ -146,6 +146,7 @@ Additional Commands: account Databricks Account Commands api Perform Databricks API call auth Authentication related commands + cache Local cache related commands completion Generate the autocompletion script for the specified shell configure Configure authentication help Help about any command From d35727a4b8819853d73f66ae4522972efb3c196a Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Thu, 25 Sep 2025 10:47:10 +0200 Subject: [PATCH 40/87] silence the first cache clear command in an acceptance test --- acceptance/cache/clear/script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/cache/clear/script b/acceptance/cache/clear/script index ada456d9513..753fdba9518 100644 --- a/acceptance/cache/clear/script +++ b/acceptance/cache/clear/script @@ -1,6 +1,6 @@ export DATABRICKS_EXPERIMENTAL_CACHE_ENABLED=true -$CLI cache clear +$CLI cache clear &> /dev/null title "First call in a session is expected to be a cache miss:\n" trace $CLI bundle validate --debug 2>&1 | grep "Local Cache" | grep -v "cache path" From 5e8fcb0c3c70570537490fd8dcbb97846aa6af9e Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Thu, 25 Sep 2025 12:51:00 +0200 Subject: [PATCH 41/87] env variable DATABRICKS_CACHE_FOLDER allows to set up location for cache files --- acceptance/cache/clear/output.txt | 2 +- acceptance/cache/clear/script | 3 +++ libs/cache/file_cache.go | 6 ++++++ libs/cache/file_cache_clear.go | 5 +++-- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/acceptance/cache/clear/output.txt b/acceptance/cache/clear/output.txt index 50239eabf68..76102c96f5c 100644 --- a/acceptance/cache/clear/output.txt +++ b/acceptance/cache/clear/output.txt @@ -14,7 +14,7 @@ [DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit: disk-read >>> [CLI] cache clear -Cache cleared successfully +Cache cleared successfully from [TEST_TMP_DIR]/.cache === First call after a clear is expected to be a cache miss: [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 diff --git a/acceptance/cache/clear/script b/acceptance/cache/clear/script index 753fdba9518..e8c6b203cc2 100644 --- a/acceptance/cache/clear/script +++ b/acceptance/cache/clear/script @@ -1,4 +1,5 @@ export DATABRICKS_EXPERIMENTAL_CACHE_ENABLED=true +export DATABRICKS_CACHE_FOLDER=$(pwd)/.cache $CLI cache clear &> /dev/null @@ -12,3 +13,5 @@ trace $CLI cache clear title "First call after a clear is expected to be a cache miss:\n" trace $CLI bundle validate --debug 2>&1 | grep "Local Cache" | grep -v "cache path" + +rm -rf "${DATABRICKS_CACHE_FOLDER}" diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index ff30e66b908..dd70d8c6e84 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -51,6 +51,12 @@ func newFileCacheWithBaseDir[T any](baseDir string, expiryMinutes int) (*FileCac } func getCacheBaseDir() (string, error) { + // Check if user has configured a custom cache directory + if customCacheDir := os.Getenv("DATABRICKS_CACHE_FOLDER"); customCacheDir != "" { + return customCacheDir, nil + } + + // Use default cache directory userCacheDir, err := os.UserCacheDir() if err != nil { return "", fmt.Errorf("failed to get user cache directory: %w", err) diff --git a/libs/cache/file_cache_clear.go b/libs/cache/file_cache_clear.go index 28a43ee3888..e3d5bc1a4bf 100644 --- a/libs/cache/file_cache_clear.go +++ b/libs/cache/file_cache_clear.go @@ -2,6 +2,7 @@ package cache import ( "context" + "fmt" "os" "github.com/databricks/cli/libs/cmdio" @@ -15,7 +16,7 @@ func ClearFileCache(ctx context.Context) error { // Check if the cache directory exists if _, err := os.Stat(databricksCacheDir); os.IsNotExist(err) { - cmdio.LogString(ctx, "No cache directory found, nothing to clear") + cmdio.LogString(ctx, fmt.Sprintf("No cache directory found at %s, nothing to clear", databricksCacheDir)) return nil } @@ -25,6 +26,6 @@ func ClearFileCache(ctx context.Context) error { return err } - cmdio.LogString(ctx, "Cache cleared successfully") + cmdio.LogString(ctx, "Cache cleared successfully from "+databricksCacheDir) return nil } From 44fc84598084835c126a851fc6fa2714ad95568f Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Thu, 25 Sep 2025 16:29:56 +0200 Subject: [PATCH 42/87] check the cache entry expiration before returning --- libs/cache/file_cache.go | 5 ++++ libs/cache/file_cache_expiry_test.go | 40 ++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index dd70d8c6e84..68738f49365 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -218,6 +218,11 @@ func (fc *FileCache[T]) readFromCache(cachePath string) (T, bool) { return zero, false } + // Check if cache entry has expired + if time.Now().After(entry.Expiry) { + return zero, false + } + var result T if err := json.Unmarshal(entry.Data, &result); err != nil { return zero, false diff --git a/libs/cache/file_cache_expiry_test.go b/libs/cache/file_cache_expiry_test.go index ab2fbb32080..deac7342e42 100644 --- a/libs/cache/file_cache_expiry_test.go +++ b/libs/cache/file_cache_expiry_test.go @@ -89,3 +89,43 @@ func TestLegacyTimestampCompatibility(t *testing.T) { assert.False(t, shouldDelete, "Legacy file should not be deleted if within MaxAge") assert.GreaterOrEqual(t, age, time.Hour, "Age should be calculated from timestamp") } + +// TestReadFromCacheRespectsExpiry tests that readFromCache returns false for expired entries +func TestReadFromCacheRespectsExpiry(t *testing.T) { + tempDir := t.TempDir() + cache, err := newFileCacheWithBaseDir[string](tempDir, 1) + require.NoError(t, err) + defer cache.StopCleanup() + + // Create an expired cache file + expiredEntry := cacheEntry{ + Data: json.RawMessage(`"expired-value"`), + Expiry: time.Now().Add(-time.Hour), // Expired 1 hour ago + } + expiredData, err := json.Marshal(expiredEntry) + require.NoError(t, err) + + expiredFile := filepath.Join(tempDir, "expired.json") + require.NoError(t, os.WriteFile(expiredFile, expiredData, 0o644)) + + // Try to read from expired cache - should return false + result, found := cache.readFromCache(expiredFile) + assert.False(t, found, "Should not find expired cache entry") + assert.Equal(t, "", result, "Result should be zero value for expired entry") + + // Create a valid (non-expired) cache file + validEntry := cacheEntry{ + Data: json.RawMessage(`"valid-value"`), + Expiry: time.Now().Add(time.Hour), // Expires in 1 hour + } + validData, err := json.Marshal(validEntry) + require.NoError(t, err) + + validFile := filepath.Join(tempDir, "valid.json") + require.NoError(t, os.WriteFile(validFile, validData, 0o644)) + + // Try to read from valid cache - should return true + result, found = cache.readFromCache(validFile) + assert.True(t, found, "Should find valid cache entry") + assert.Equal(t, "valid-value", result, "Should return correct value for valid entry") +} From 8c0afb89af9e703512c0af6fd1d80fcfb4f41c29 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Thu, 25 Sep 2025 16:48:04 +0200 Subject: [PATCH 43/87] turn the cache to be always on; DATABRICKS_CACHE_DISABLED env variable to turn it off --- acceptance/cache/clear/script | 1 - acceptance/cache/exploratory/script | 2 -- bundle/config/mutator/populate_current_user.go | 4 ++-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/acceptance/cache/clear/script b/acceptance/cache/clear/script index e8c6b203cc2..076fbd722b4 100644 --- a/acceptance/cache/clear/script +++ b/acceptance/cache/clear/script @@ -1,4 +1,3 @@ -export DATABRICKS_EXPERIMENTAL_CACHE_ENABLED=true export DATABRICKS_CACHE_FOLDER=$(pwd)/.cache $CLI cache clear &> /dev/null diff --git a/acceptance/cache/exploratory/script b/acceptance/cache/exploratory/script index 5646a4f1263..602b9bcbc4b 100644 --- a/acceptance/cache/exploratory/script +++ b/acceptance/cache/exploratory/script @@ -1,5 +1,3 @@ -export DATABRICKS_EXPERIMENTAL_CACHE_ENABLED=true - title "First call in a session is expected to be a cache miss:\n" trace $CLI bundle validate -p dogfood --debug 2>&1 | grep "Local Cache" | grep -v "cache path" diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index dd634206781..067009ba342 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -33,8 +33,8 @@ func (m *populateCurrentUser) initializeCache(ctx context.Context, b *bundle.Bun return } - if os.Getenv("DATABRICKS_EXPERIMENTAL_CACHE_ENABLED") != "true" { - log.Debugf(ctx, "[Local Cache] Local cache is disabled. Enable it be setting an env variable DATABRICKS_EXPERIMENTAL_CACHE_ENABLED=true\n") + if os.Getenv("DATABRICKS_CACHE_DISABLED") == "true" { + log.Debugf(ctx, "[Local Cache] Local cache is disabled via environment variable DATABRICKS_CACHE_DISABLED=true\n") return } From a374a464d90e20ee0e8914d35c4c859bc33ada69 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Fri, 26 Sep 2025 15:40:03 +0200 Subject: [PATCH 44/87] make cache hits no-op --- libs/cache/file_cache.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 68738f49365..783d19c20eb 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -116,11 +116,11 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu // Check in-memory cache first fc.mu.RLock() - if data, found := fc.memCache[cacheKey]; found { + if _, found := fc.memCache[cacheKey]; found { fc.mu.RUnlock() log.Debugf(ctx, "[Local Cache] cache hit: in-memory\n") fc.addTelemetryMetric("local.cache.hit") - return data, nil + // return data, nil // cache layer is currently no-op } fc.mu.RUnlock() @@ -132,7 +132,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu fc.mu.Unlock() log.Debugf(ctx, "[Local Cache] cache hit: disk-read\n") fc.addTelemetryMetric("local.cache.hit") - return data, nil + // return data, nil // cache layer is currently no-op } // Check if there's a pending write for this key @@ -144,11 +144,11 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu case <-pendingCh: // Try reading from memory cache again fc.mu.RLock() - if data, found := fc.memCache[cacheKey]; found { + if _, found := fc.memCache[cacheKey]; found { fc.mu.RUnlock() log.Debugf(ctx, "[Local Cache] cache hit: in-memory from pending write\n") fc.addTelemetryMetric("local.cache.hit") - return data, nil + // return data, nil // cache layer is currently no-op } fc.mu.RUnlock() case <-ctx.Done(): From f6d1807ae5cef625a01cdafb0abe36c2b505e11f Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Fri, 26 Sep 2025 15:55:34 +0200 Subject: [PATCH 45/87] make test-update --- .../bundle/resource_deps/job_tasks/output.txt | 3 +++ .../resource_deps/resources_var/output.txt | 3 +++ .../telemetry/deploy-compute-type/output.txt | 24 +++++++++++++++++++ .../telemetry/deploy-experimental/output.txt | 12 ++++++++++ .../deploy-name-prefix/custom/output.txt | 12 ++++++++++ .../mode-development/output.txt | 12 ++++++++++ .../telemetry/deploy-whl-artifacts/output.txt | 24 +++++++++++++++++++ .../bundle/telemetry/deploy/out.telemetry.txt | 12 ++++++++++ acceptance/cache/clear/output.txt | 1 + acceptance/cache/exploratory/output.txt | 3 +++ 10 files changed, 106 insertions(+) diff --git a/acceptance/bundle/resource_deps/job_tasks/output.txt b/acceptance/bundle/resource_deps/job_tasks/output.txt index fb6f4905b90..80921769a81 100644 --- a/acceptance/bundle/resource_deps/job_tasks/output.txt +++ b/acceptance/bundle/resource_deps/job_tasks/output.txt @@ -11,6 +11,9 @@ experimental.use_legacy_run_as false has_classic_interactive_compute false has_classic_job_compute false has_serverless_compute true +local.cache.attempt true +local.cache.hit true +local.cache.miss true presets_name_prefix_is_set false python_wheel_wrapper_is_set false resref_jobs.tags.* true diff --git a/acceptance/bundle/resource_deps/resources_var/output.txt b/acceptance/bundle/resource_deps/resources_var/output.txt index 2454bf669f0..f559331a9c5 100644 --- a/acceptance/bundle/resource_deps/resources_var/output.txt +++ b/acceptance/bundle/resource_deps/resources_var/output.txt @@ -41,6 +41,9 @@ experimental.use_legacy_run_as false has_classic_interactive_compute false has_classic_job_compute false has_serverless_compute false +local.cache.attempt true +local.cache.hit true +local.cache.miss true presets_name_prefix_is_set true python_wheel_wrapper_is_set false resref_volumes.catalog_name true diff --git a/acceptance/bundle/telemetry/deploy-compute-type/output.txt b/acceptance/bundle/telemetry/deploy-compute-type/output.txt index f6376bf9dd3..ed86c40e921 100644 --- a/acceptance/bundle/telemetry/deploy-compute-type/output.txt +++ b/acceptance/bundle/telemetry/deploy-compute-type/output.txt @@ -13,6 +13,18 @@ Deployment complete! >>> cat out.requests.txt [ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.hit", + "value": true + }, + { + "key": "local.cache.miss", + "value": true + }, { "key": "experimental.use_legacy_run_as", "value": false @@ -47,6 +59,18 @@ Deployment complete! } ] [ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.hit", + "value": true + }, + { + "key": "local.cache.miss", + "value": true + }, { "key": "experimental.use_legacy_run_as", "value": false diff --git a/acceptance/bundle/telemetry/deploy-experimental/output.txt b/acceptance/bundle/telemetry/deploy-experimental/output.txt index 437a3c6f9e9..7d2c4dfb8c9 100644 --- a/acceptance/bundle/telemetry/deploy-experimental/output.txt +++ b/acceptance/bundle/telemetry/deploy-experimental/output.txt @@ -12,6 +12,18 @@ Deployment complete! >>> cat out.requests.txt { "bool_values": [ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.hit", + "value": true + }, + { + "key": "local.cache.miss", + "value": true + }, { "key": "experimental.use_legacy_run_as", "value": true diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt b/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt index 567b4282000..eb0ac9dbc39 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt @@ -8,6 +8,18 @@ Deployment complete! >>> cat out.requests.txt { "bool_values": [ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.hit", + "value": true + }, + { + "key": "local.cache.miss", + "value": true + }, { "key": "experimental.use_legacy_run_as", "value": false diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt index 7a710b9045b..36dec6a5084 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt @@ -8,6 +8,18 @@ Deployment complete! >>> cat out.requests.txt { "bool_values": [ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.hit", + "value": true + }, + { + "key": "local.cache.miss", + "value": true + }, { "key": "experimental.use_legacy_run_as", "value": false diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt b/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt index 207ee71d24c..9cac927b1ac 100644 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt @@ -16,6 +16,18 @@ Deployment complete! >>> cat out.requests.txt { "bool_values": [ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.hit", + "value": true + }, + { + "key": "local.cache.miss", + "value": true + }, { "key": "artifact_build_command_is_set", "value": false @@ -48,6 +60,18 @@ Deployment complete! } { "bool_values": [ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.hit", + "value": true + }, + { + "key": "local.cache.miss", + "value": true + }, { "key": "artifact_build_command_is_set", "value": true diff --git a/acceptance/bundle/telemetry/deploy/out.telemetry.txt b/acceptance/bundle/telemetry/deploy/out.telemetry.txt index 57b9b46f855..5510ea686f1 100644 --- a/acceptance/bundle/telemetry/deploy/out.telemetry.txt +++ b/acceptance/bundle/telemetry/deploy/out.telemetry.txt @@ -42,6 +42,18 @@ "lookup_variable_count": 0, "target_count": 1, "bool_values": [ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.hit", + "value": true + }, + { + "key": "local.cache.miss", + "value": true + }, { "key": "experimental.use_legacy_run_as", "value": false diff --git a/acceptance/cache/clear/output.txt b/acceptance/cache/clear/output.txt index 76102c96f5c..efa74d102be 100644 --- a/acceptance/cache/clear/output.txt +++ b/acceptance/cache/clear/output.txt @@ -12,6 +12,7 @@ [DEBUG_TIMESTAMP] Debug: [Local Cache] using fingerprint with hash: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit: disk-read +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, but stored the compute result for future calls >>> [CLI] cache clear Cache cleared successfully from [TEST_TMP_DIR]/.cache diff --git a/acceptance/cache/exploratory/output.txt b/acceptance/cache/exploratory/output.txt index 9fda4f8ddaf..1039bfb60a6 100644 --- a/acceptance/cache/exploratory/output.txt +++ b/acceptance/cache/exploratory/output.txt @@ -4,6 +4,7 @@ [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using fingerprint with hash: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit: disk-read [DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, but stored the compute result for future calls === Second call in a session is expected to be a cache hit @@ -12,6 +13,7 @@ [DEBUG_TIMESTAMP] Debug: [Local Cache] using fingerprint with hash: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit: disk-read +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, but stored the compute result for future calls === Bundle deploy should send telemetry values @@ -23,3 +25,4 @@ Deployment complete! >>> print_telemetry_bool_values local.cache.attempt true local.cache.hit true +local.cache.miss true From e31f7fd9d9e7eab45b9ff387c6e5b85c41349c8f Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Fri, 26 Sep 2025 16:14:58 +0200 Subject: [PATCH 46/87] fix mutex unlocking --- libs/cache/file_cache.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 783d19c20eb..c8fea42a7b3 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -121,8 +121,9 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu log.Debugf(ctx, "[Local Cache] cache hit: in-memory\n") fc.addTelemetryMetric("local.cache.hit") // return data, nil // cache layer is currently no-op + } else { + fc.mu.RUnlock() } - fc.mu.RUnlock() // Try to read from disk cache if data, found := fc.readFromCache(cachePath); found { @@ -149,8 +150,9 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu log.Debugf(ctx, "[Local Cache] cache hit: in-memory from pending write\n") fc.addTelemetryMetric("local.cache.hit") // return data, nil // cache layer is currently no-op + } else { + fc.mu.RUnlock() } - fc.mu.RUnlock() case <-ctx.Done(): log.Debugf(ctx, "[Local Cache] cache miss: no hit while waiting for pending write\n") fc.addTelemetryMetric("local.cache.miss") From f8287ad10759a42429699a290ac1b666ddc4b695 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Fri, 26 Sep 2025 16:35:56 +0200 Subject: [PATCH 47/87] update unit tests for no-op caching layer --- libs/cache/file_cache_test.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index 8042e33a2ed..657a974b175 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -100,8 +100,10 @@ func TestFileCacheGetOrCompute(t *testing.T) { }) require.NoError(t, err) - assert.Equal(t, expectedValue, result2) - assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) // Should still be 1 + // File cache makes the second call while cache layer is no-op: + assert.Equal(t, "should-not-be-called", result2) + // assert.Equal(t, expectedValue, result2) + assert.Equal(t, int32(2), atomic.LoadInt32(&computeCalls)) // Allow time for async writes to complete before test cleanup time.Sleep(50 * time.Millisecond) @@ -168,8 +170,8 @@ func TestFileCacheGetOrComputeConcurrency(t *testing.T) { assert.Equal(t, expectedValue, result) } - // Compute should have been called only once despite multiple concurrent requests - assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) + // Compute is called 10 times while cache layer is no-op: + assert.Equal(t, int32(10), atomic.LoadInt32(&computeCalls)) // Allow time for async writes to complete before test cleanup time.Sleep(50 * time.Millisecond) @@ -242,8 +244,10 @@ func TestFingerprintDeterministic(t *testing.T) { return "should-not-be-called", nil }) require.NoError(t, err) - assert.Equal(t, expectedValue, result2) - assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) // Should still be 1 + + // File cache makes the second call while cache layer is no-op: + assert.Equal(t, "should-not-be-called", result2) + assert.Equal(t, int32(2), atomic.LoadInt32(&computeCalls)) // Should still be 1 // Allow time for async writes to complete before test cleanup time.Sleep(50 * time.Millisecond) From 5f1b8653b77ccd4d5d126ee4f414588a5b3a64df Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Fri, 26 Sep 2025 17:17:02 +0200 Subject: [PATCH 48/87] use unique cache folder in each acceptance test --- acceptance/acceptance_test.go | 6 ++++++ .../bundle/resource_deps/job_tasks/output.txt | 1 - .../bundle/resource_deps/resources_var/output.txt | 1 - .../bundle/telemetry/deploy-compute-type/output.txt | 8 -------- .../bundle/telemetry/deploy-experimental/output.txt | 4 ---- .../telemetry/deploy-name-prefix/custom/output.txt | 4 ---- .../deploy-name-prefix/mode-development/output.txt | 4 ---- .../telemetry/deploy-whl-artifacts/output.txt | 8 -------- .../bundle/telemetry/deploy/out.telemetry.txt | 4 ---- acceptance/cache/exploratory/output.txt | 2 -- libs/cache/file_cache.go | 13 +++++++++++-- 11 files changed, 17 insertions(+), 38 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index c776645fa8d..65502659d68 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -598,6 +598,12 @@ func runTest(t *testing.T, envBase := getCloudEnvBase(cloudEnv) cmd.Env = append(cmd.Env, "CLOUD_ENV_BASE="+envBase) + // Set unique cache folder for this test to avoid race conditions between parallel tests + userCacheDir, err := os.UserCacheDir() + require.NoError(t, err) + uniqueCacheDir := filepath.Join(userCacheDir, "databricks-test-"+uniqueName) + cmd.Env = append(cmd.Env, "DATABRICKS_CACHE_FOLDER="+uniqueCacheDir) + // Must be added PrepareReplacementsUser, otherwise conflicts with [USERNAME] testdiff.PrepareReplacementsUUID(t, &repls) diff --git a/acceptance/bundle/resource_deps/job_tasks/output.txt b/acceptance/bundle/resource_deps/job_tasks/output.txt index 80921769a81..64b9a619f74 100644 --- a/acceptance/bundle/resource_deps/job_tasks/output.txt +++ b/acceptance/bundle/resource_deps/job_tasks/output.txt @@ -12,7 +12,6 @@ has_classic_interactive_compute false has_classic_job_compute false has_serverless_compute true local.cache.attempt true -local.cache.hit true local.cache.miss true presets_name_prefix_is_set false python_wheel_wrapper_is_set false diff --git a/acceptance/bundle/resource_deps/resources_var/output.txt b/acceptance/bundle/resource_deps/resources_var/output.txt index f559331a9c5..6ea6012382a 100644 --- a/acceptance/bundle/resource_deps/resources_var/output.txt +++ b/acceptance/bundle/resource_deps/resources_var/output.txt @@ -43,7 +43,6 @@ has_classic_job_compute false has_serverless_compute false local.cache.attempt true local.cache.hit true -local.cache.miss true presets_name_prefix_is_set true python_wheel_wrapper_is_set false resref_volumes.catalog_name true diff --git a/acceptance/bundle/telemetry/deploy-compute-type/output.txt b/acceptance/bundle/telemetry/deploy-compute-type/output.txt index ed86c40e921..a424df8ce1d 100644 --- a/acceptance/bundle/telemetry/deploy-compute-type/output.txt +++ b/acceptance/bundle/telemetry/deploy-compute-type/output.txt @@ -17,10 +17,6 @@ Deployment complete! "key": "local.cache.attempt", "value": true }, - { - "key": "local.cache.hit", - "value": true - }, { "key": "local.cache.miss", "value": true @@ -67,10 +63,6 @@ Deployment complete! "key": "local.cache.hit", "value": true }, - { - "key": "local.cache.miss", - "value": true - }, { "key": "experimental.use_legacy_run_as", "value": false diff --git a/acceptance/bundle/telemetry/deploy-experimental/output.txt b/acceptance/bundle/telemetry/deploy-experimental/output.txt index 7d2c4dfb8c9..05bc64e441d 100644 --- a/acceptance/bundle/telemetry/deploy-experimental/output.txt +++ b/acceptance/bundle/telemetry/deploy-experimental/output.txt @@ -16,10 +16,6 @@ Deployment complete! "key": "local.cache.attempt", "value": true }, - { - "key": "local.cache.hit", - "value": true - }, { "key": "local.cache.miss", "value": true diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt b/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt index eb0ac9dbc39..31ff8e9cf7e 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt @@ -12,10 +12,6 @@ Deployment complete! "key": "local.cache.attempt", "value": true }, - { - "key": "local.cache.hit", - "value": true - }, { "key": "local.cache.miss", "value": true diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt index 36dec6a5084..39b671bec32 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt @@ -12,10 +12,6 @@ Deployment complete! "key": "local.cache.attempt", "value": true }, - { - "key": "local.cache.hit", - "value": true - }, { "key": "local.cache.miss", "value": true diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt b/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt index 9cac927b1ac..a9b8ce4ae6e 100644 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt @@ -20,10 +20,6 @@ Deployment complete! "key": "local.cache.attempt", "value": true }, - { - "key": "local.cache.hit", - "value": true - }, { "key": "local.cache.miss", "value": true @@ -68,10 +64,6 @@ Deployment complete! "key": "local.cache.hit", "value": true }, - { - "key": "local.cache.miss", - "value": true - }, { "key": "artifact_build_command_is_set", "value": true diff --git a/acceptance/bundle/telemetry/deploy/out.telemetry.txt b/acceptance/bundle/telemetry/deploy/out.telemetry.txt index 5510ea686f1..48e0e8e77c2 100644 --- a/acceptance/bundle/telemetry/deploy/out.telemetry.txt +++ b/acceptance/bundle/telemetry/deploy/out.telemetry.txt @@ -46,10 +46,6 @@ "key": "local.cache.attempt", "value": true }, - { - "key": "local.cache.hit", - "value": true - }, { "key": "local.cache.miss", "value": true diff --git a/acceptance/cache/exploratory/output.txt b/acceptance/cache/exploratory/output.txt index 1039bfb60a6..4272a06a49f 100644 --- a/acceptance/cache/exploratory/output.txt +++ b/acceptance/cache/exploratory/output.txt @@ -4,7 +4,6 @@ [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using fingerprint with hash: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit: disk-read [DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, but stored the compute result for future calls === Second call in a session is expected to be a cache hit @@ -25,4 +24,3 @@ Deployment complete! >>> print_telemetry_bool_values local.cache.attempt true local.cache.hit true -local.cache.miss true diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index c8fea42a7b3..92d96b7aee9 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -96,6 +96,7 @@ func (fc *FileCache[T]) addTelemetryMetric(key string) { // GetOrCompute retrieves cached content or computes it using the provided function. func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compute func(ctx context.Context) (T, error)) (T, error) { var zero T + isCacheHit := false // Convert fingerprint to deterministic string fingerprintHash, err := fingerprintToHash(fingerprint) @@ -120,6 +121,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu fc.mu.RUnlock() log.Debugf(ctx, "[Local Cache] cache hit: in-memory\n") fc.addTelemetryMetric("local.cache.hit") + isCacheHit = true // return data, nil // cache layer is currently no-op } else { fc.mu.RUnlock() @@ -133,6 +135,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu fc.mu.Unlock() log.Debugf(ctx, "[Local Cache] cache hit: disk-read\n") fc.addTelemetryMetric("local.cache.hit") + isCacheHit = true // return data, nil // cache layer is currently no-op } @@ -149,6 +152,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu fc.mu.RUnlock() log.Debugf(ctx, "[Local Cache] cache hit: in-memory from pending write\n") fc.addTelemetryMetric("local.cache.hit") + isCacheHit = true // return data, nil // cache layer is currently no-op } else { fc.mu.RUnlock() @@ -176,7 +180,9 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu select { case <-ctx.Done(): log.Debugf(ctx, "[Local Cache] cache miss: context is already cancelled\n") - fc.addTelemetryMetric("local.cache.miss") + if !isCacheHit { + fc.addTelemetryMetric("local.cache.miss") + } return zero, ctx.Err() default: } @@ -199,7 +205,10 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu go fc.writeToCache(cachePath, result) log.Debugf(ctx, "[Local Cache] cache miss, but stored the compute result for future calls\n") - fc.addTelemetryMetric("local.cache.miss") + + if !isCacheHit { + fc.addTelemetryMetric("local.cache.miss") + } return result, nil } From ef495c334982ba7590c7118a8b89b37020c3208b Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Fri, 26 Sep 2025 18:06:18 +0200 Subject: [PATCH 49/87] disable caching for telemetry acc tests --- acceptance/acceptance_test.go | 12 ++++++------ .../telemetry/deploy-compute-type/output.txt | 16 ---------------- .../telemetry/deploy-experimental/output.txt | 8 -------- .../deploy-name-prefix/custom/output.txt | 8 -------- .../mode-development/output.txt | 8 -------- .../telemetry/deploy-whl-artifacts/output.txt | 16 ---------------- .../bundle/telemetry/deploy/out.telemetry.txt | 8 -------- acceptance/bundle/telemetry/test.toml | 3 +++ .../cache/{exploratory => simple}/databricks.yml | 0 .../cache/{exploratory => simple}/out.test.toml | 0 .../cache/{exploratory => simple}/output.txt | 0 acceptance/cache/{exploratory => simple}/script | 5 +++++ .../cache/{exploratory => simple}/test.toml | 0 13 files changed, 14 insertions(+), 70 deletions(-) rename acceptance/cache/{exploratory => simple}/databricks.yml (100%) rename acceptance/cache/{exploratory => simple}/out.test.toml (100%) rename acceptance/cache/{exploratory => simple}/output.txt (100%) rename acceptance/cache/{exploratory => simple}/script (81%) rename acceptance/cache/{exploratory => simple}/test.toml (100%) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 65502659d68..668f8e12fd2 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -598,12 +598,6 @@ func runTest(t *testing.T, envBase := getCloudEnvBase(cloudEnv) cmd.Env = append(cmd.Env, "CLOUD_ENV_BASE="+envBase) - // Set unique cache folder for this test to avoid race conditions between parallel tests - userCacheDir, err := os.UserCacheDir() - require.NoError(t, err) - uniqueCacheDir := filepath.Join(userCacheDir, "databricks-test-"+uniqueName) - cmd.Env = append(cmd.Env, "DATABRICKS_CACHE_FOLDER="+uniqueCacheDir) - // Must be added PrepareReplacementsUser, otherwise conflicts with [USERNAME] testdiff.PrepareReplacementsUUID(t, &repls) @@ -629,6 +623,12 @@ func runTest(t *testing.T, cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir) } + // Set unique cache folder for this test to avoid race conditions between parallel tests + userCacheDir, err := os.UserCacheDir() + require.NoError(t, err) + uniqueCacheDir := filepath.Join(userCacheDir, strings.ReplaceAll(dir, string(os.PathSeparator), "--")) + cmd.Env = append(cmd.Env, "DATABRICKS_CACHE_FOLDER="+uniqueCacheDir) + for _, key := range utils.SortedKeys(config.Env) { if hasKey(customEnv, key) { // We want EnvMatrix to take precedence. diff --git a/acceptance/bundle/telemetry/deploy-compute-type/output.txt b/acceptance/bundle/telemetry/deploy-compute-type/output.txt index a424df8ce1d..f6376bf9dd3 100644 --- a/acceptance/bundle/telemetry/deploy-compute-type/output.txt +++ b/acceptance/bundle/telemetry/deploy-compute-type/output.txt @@ -13,14 +13,6 @@ Deployment complete! >>> cat out.requests.txt [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, { "key": "experimental.use_legacy_run_as", "value": false @@ -55,14 +47,6 @@ Deployment complete! } ] [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.hit", - "value": true - }, { "key": "experimental.use_legacy_run_as", "value": false diff --git a/acceptance/bundle/telemetry/deploy-experimental/output.txt b/acceptance/bundle/telemetry/deploy-experimental/output.txt index 05bc64e441d..437a3c6f9e9 100644 --- a/acceptance/bundle/telemetry/deploy-experimental/output.txt +++ b/acceptance/bundle/telemetry/deploy-experimental/output.txt @@ -12,14 +12,6 @@ Deployment complete! >>> cat out.requests.txt { "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, { "key": "experimental.use_legacy_run_as", "value": true diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt b/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt index 31ff8e9cf7e..567b4282000 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt @@ -8,14 +8,6 @@ Deployment complete! >>> cat out.requests.txt { "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, { "key": "experimental.use_legacy_run_as", "value": false diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt index 39b671bec32..7a710b9045b 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt @@ -8,14 +8,6 @@ Deployment complete! >>> cat out.requests.txt { "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, { "key": "experimental.use_legacy_run_as", "value": false diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt b/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt index a9b8ce4ae6e..207ee71d24c 100644 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt @@ -16,14 +16,6 @@ Deployment complete! >>> cat out.requests.txt { "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, { "key": "artifact_build_command_is_set", "value": false @@ -56,14 +48,6 @@ Deployment complete! } { "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.hit", - "value": true - }, { "key": "artifact_build_command_is_set", "value": true diff --git a/acceptance/bundle/telemetry/deploy/out.telemetry.txt b/acceptance/bundle/telemetry/deploy/out.telemetry.txt index 48e0e8e77c2..57b9b46f855 100644 --- a/acceptance/bundle/telemetry/deploy/out.telemetry.txt +++ b/acceptance/bundle/telemetry/deploy/out.telemetry.txt @@ -42,14 +42,6 @@ "lookup_variable_count": 0, "target_count": 1, "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, { "key": "experimental.use_legacy_run_as", "value": false diff --git a/acceptance/bundle/telemetry/test.toml b/acceptance/bundle/telemetry/test.toml index d583d9d69b0..9bdbed40ebc 100644 --- a/acceptance/bundle/telemetry/test.toml +++ b/acceptance/bundle/telemetry/test.toml @@ -1,6 +1,9 @@ RecordRequests = true IncludeRequestHeaders = ["User-Agent"] +[Env] +DATABRICKS_CACHE_DISABLED = 'true' + [[Repls]] Old = '"execution_time_ms": \d{1,5},' New = '"execution_time_ms": SMALL_INT,' diff --git a/acceptance/cache/exploratory/databricks.yml b/acceptance/cache/simple/databricks.yml similarity index 100% rename from acceptance/cache/exploratory/databricks.yml rename to acceptance/cache/simple/databricks.yml diff --git a/acceptance/cache/exploratory/out.test.toml b/acceptance/cache/simple/out.test.toml similarity index 100% rename from acceptance/cache/exploratory/out.test.toml rename to acceptance/cache/simple/out.test.toml diff --git a/acceptance/cache/exploratory/output.txt b/acceptance/cache/simple/output.txt similarity index 100% rename from acceptance/cache/exploratory/output.txt rename to acceptance/cache/simple/output.txt diff --git a/acceptance/cache/exploratory/script b/acceptance/cache/simple/script similarity index 81% rename from acceptance/cache/exploratory/script rename to acceptance/cache/simple/script index 602b9bcbc4b..6a1f1de5a6e 100644 --- a/acceptance/cache/exploratory/script +++ b/acceptance/cache/simple/script @@ -1,3 +1,7 @@ +export DATABRICKS_CACHE_FOLDER=$(pwd)/.cache + +$CLI cache clear &> /dev/null + title "First call in a session is expected to be a cache miss:\n" trace $CLI bundle validate -p dogfood --debug 2>&1 | grep "Local Cache" | grep -v "cache path" @@ -9,3 +13,4 @@ trace $CLI bundle deploy -p dogfood trace print_telemetry_bool_values | grep "local.cache" rm out.requests.txt +rm -rf "${DATABRICKS_CACHE_FOLDER}" diff --git a/acceptance/cache/exploratory/test.toml b/acceptance/cache/simple/test.toml similarity index 100% rename from acceptance/cache/exploratory/test.toml rename to acceptance/cache/simple/test.toml From 782c5b16ce6c008f6c9948b82b2b4565b4324724 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 4 Nov 2025 11:47:22 +0100 Subject: [PATCH 50/87] fix sync code --- libs/cache/file_cache.go | 134 ++++++++++++++-------------------- libs/cache/file_cache_test.go | 6 +- 2 files changed, 57 insertions(+), 83 deletions(-) diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 92d96b7aee9..4ba9149278a 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -3,7 +3,6 @@ package cache import ( "context" "crypto/rand" - "crypto/sha256" "encoding/hex" "encoding/json" "fmt" @@ -22,10 +21,10 @@ type FileCache[T any] struct { baseDir string expiryMinutes int mu sync.RWMutex - pending map[string]chan struct{} // Track pending writes - memCache map[string]T // In-memory cache for immediate access - cleanupMgr *CleanupManager // Background cleanup manager - metrics *bundle.Metrics // Telemetry metrics + computeOnce map[string]*sync.Once // Ensure only one goroutine computes per key + memCache map[string]T // In-memory cache for immediate access + cleanupMgr *CleanupManager // Background cleanup manager + metrics *bundle.Metrics // Telemetry metrics } // newFileCacheWithBaseDir creates a new file-based cache that stores data in the specified directory. @@ -39,7 +38,7 @@ func newFileCacheWithBaseDir[T any](baseDir string, expiryMinutes int) (*FileCac fc := &FileCache[T]{ baseDir: baseDir, expiryMinutes: expiryMinutes, - pending: make(map[string]chan struct{}), + computeOnce: make(map[string]*sync.Once), memCache: make(map[string]T), cleanupMgr: cleanupMgr, } @@ -96,11 +95,10 @@ func (fc *FileCache[T]) addTelemetryMetric(key string) { // GetOrCompute retrieves cached content or computes it using the provided function. func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compute func(ctx context.Context) (T, error)) (T, error) { var zero T - isCacheHit := false - // Convert fingerprint to deterministic string - fingerprintHash, err := fingerprintToHash(fingerprint) - log.Debugf(ctx, "[Local Cache] using fingerprint with hash: %s\n", fingerprintHash) + // Convert fingerprint to deterministic hash - this is our cache key + cacheKey, err := fingerprintToHash(fingerprint) + log.Debugf(ctx, "[Local Cache] using cache key: %s\n", cacheKey) fc.addTelemetryMetric("local.cache.attempt") @@ -109,9 +107,6 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu return zero, fmt.Errorf("failed to convert fingerprint to string: %w", err) } - cacheKey := fc.getCacheKey(fingerprintHash) - log.Debugf(ctx, "[Local Cache] using cache key: %s\n", cacheKey) - cachePath := fc.getCachePath(cacheKey) log.Debugf(ctx, "[Local Cache] using cache path: %s\n", cachePath) @@ -121,7 +116,6 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu fc.mu.RUnlock() log.Debugf(ctx, "[Local Cache] cache hit: in-memory\n") fc.addTelemetryMetric("local.cache.hit") - isCacheHit = true // return data, nil // cache layer is currently no-op } else { fc.mu.RUnlock() @@ -135,80 +129,69 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu fc.mu.Unlock() log.Debugf(ctx, "[Local Cache] cache hit: disk-read\n") fc.addTelemetryMetric("local.cache.hit") - isCacheHit = true // return data, nil // cache layer is currently no-op } - // Check if there's a pending write for this key + // Get or create sync.Once for this cache key fc.mu.Lock() - if pendingCh, exists := fc.pending[cacheKey]; exists { - fc.mu.Unlock() - // Wait for pending write to complete + once, exists := fc.computeOnce[cacheKey] + if !exists { + once = &sync.Once{} + fc.computeOnce[cacheKey] = once + } + fc.mu.Unlock() + + // Use sync.Once to ensure only one goroutine computes the value + var result T + var computeErr error + once.Do(func() { + // Check if context is already cancelled before computing select { - case <-pendingCh: - // Try reading from memory cache again - fc.mu.RLock() - if _, found := fc.memCache[cacheKey]; found { - fc.mu.RUnlock() - log.Debugf(ctx, "[Local Cache] cache hit: in-memory from pending write\n") - fc.addTelemetryMetric("local.cache.hit") - isCacheHit = true - // return data, nil // cache layer is currently no-op - } else { - fc.mu.RUnlock() - } case <-ctx.Done(): - log.Debugf(ctx, "[Local Cache] cache miss: no hit while waiting for pending write\n") + log.Debugf(ctx, "[Local Cache] cache miss: context is already cancelled\n") fc.addTelemetryMetric("local.cache.miss") - return zero, ctx.Err() + computeErr = ctx.Err() + return + default: } - } else { - // Mark this key as pending - pendingCh := make(chan struct{}) - fc.pending[cacheKey] = pendingCh - fc.mu.Unlock() - - defer func() { - fc.mu.Lock() - delete(fc.pending, cacheKey) - close(pendingCh) - fc.mu.Unlock() - }() - } - // Check if context is already cancelled before computing - select { - case <-ctx.Done(): - log.Debugf(ctx, "[Local Cache] cache miss: context is already cancelled\n") - if !isCacheHit { - fc.addTelemetryMetric("local.cache.miss") + // Compute the value + result, computeErr = compute(ctx) + if computeErr != nil { + log.Debugf(ctx, "[Local Cache] error while caching: %v\n", computeErr) + fc.addTelemetryMetric("local.cache.error") + return } - return zero, ctx.Err() - default: - } - // Compute the value - result, err := compute(ctx) - if err != nil { - log.Debugf(ctx, "[Local Cache] error while caching: %v\n", err) - fc.addTelemetryMetric("local.cache.error") - return zero, err - } + // Store in memory cache immediately + fc.mu.Lock() + fc.memCache[cacheKey] = result + fc.mu.Unlock() + + // Async write to disk cache + log.Debugf(ctx, "[Local Cache] async writing to cache path: %s\n", cachePath) + go fc.writeToCache(cachePath, result) - // Store in memory cache immediately + log.Debugf(ctx, "[Local Cache] cache miss, but stored the compute result for future calls\n") + fc.addTelemetryMetric("local.cache.miss") + }) + + // Clean up the sync.Once instance after use to prevent memory leaks fc.mu.Lock() - fc.memCache[cacheKey] = result + delete(fc.computeOnce, cacheKey) fc.mu.Unlock() - // Async write to disk cache - log.Debugf(ctx, "[Local Cache] async writing to cache path: %s\n", cachePath) - go fc.writeToCache(cachePath, result) - - log.Debugf(ctx, "[Local Cache] cache miss, but stored the compute result for future calls\n") + if computeErr != nil { + return zero, computeErr + } - if !isCacheHit { - fc.addTelemetryMetric("local.cache.miss") + // If another goroutine computed the value, retrieve it from memory cache + fc.mu.RLock() + if data, found := fc.memCache[cacheKey]; found { + result = data } + fc.mu.RUnlock() + return result, nil } @@ -216,9 +199,6 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu func (fc *FileCache[T]) readFromCache(cachePath string) (T, bool) { var zero T - fc.mu.RLock() - defer fc.mu.RUnlock() - data, err := os.ReadFile(cachePath) if err != nil { return zero, false @@ -289,12 +269,6 @@ func generateTempPath(cachePath string) (string, error) { return cachePath + ".tmp." + randomSuffix, nil } -// getCacheKey generates a safe cache key from the fingerprint. -func (fc *FileCache[T]) getCacheKey(fingerprint string) string { - hash := sha256.Sum256([]byte(fingerprint)) - return hex.EncodeToString(hash[:]) -} - // getCachePath returns the full path to the cache file for a given cache key. func (fc *FileCache[T]) getCachePath(cacheKey string) string { return filepath.Join(fc.baseDir, cacheKey+".json") diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index 657a974b175..2b98386a8f0 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -21,7 +21,7 @@ func TestNewFileCache(t *testing.T) { require.NoError(t, err) assert.NotNil(t, cache) assert.Equal(t, cacheDir, cache.baseDir) - assert.NotNil(t, cache.pending) + assert.NotNil(t, cache.computeOnce) // Verify directory was created info, err := os.Stat(cacheDir) @@ -170,8 +170,8 @@ func TestFileCacheGetOrComputeConcurrency(t *testing.T) { assert.Equal(t, expectedValue, result) } - // Compute is called 10 times while cache layer is no-op: - assert.Equal(t, int32(10), atomic.LoadInt32(&computeCalls)) + // With sync.Once, compute should only be called once even with concurrent requests + assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) // Allow time for async writes to complete before test cleanup time.Sleep(50 * time.Millisecond) From e4f03c2de3cb302bbbe07ff8040e1097594cb8bb Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 4 Nov 2025 11:58:39 +0100 Subject: [PATCH 51/87] simplify --- libs/cache/file_cache.go | 60 ++++++++++++++++++----------------- libs/cache/file_cache_test.go | 15 ++++----- 2 files changed, 37 insertions(+), 38 deletions(-) diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 4ba9149278a..204f3b5e272 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -98,28 +98,24 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu // Convert fingerprint to deterministic hash - this is our cache key cacheKey, err := fingerprintToHash(fingerprint) - log.Debugf(ctx, "[Local Cache] using cache key: %s\n", cacheKey) - - fc.addTelemetryMetric("local.cache.attempt") - if err != nil { - log.Debugf(ctx, "[Local Cache] cache miss: non-compliant fingerprint\n") return zero, fmt.Errorf("failed to convert fingerprint to string: %w", err) } + log.Debugf(ctx, "[Local Cache] using cache key: %s\n", cacheKey) + fc.addTelemetryMetric("local.cache.attempt") + cachePath := fc.getCachePath(cacheKey) - log.Debugf(ctx, "[Local Cache] using cache path: %s\n", cachePath) - // Check in-memory cache first + // Check in-memory cache first (fast path) fc.mu.RLock() - if _, found := fc.memCache[cacheKey]; found { + if data, found := fc.memCache[cacheKey]; found { fc.mu.RUnlock() log.Debugf(ctx, "[Local Cache] cache hit: in-memory\n") fc.addTelemetryMetric("local.cache.hit") - // return data, nil // cache layer is currently no-op - } else { - fc.mu.RUnlock() + return data, nil } + fc.mu.RUnlock() // Try to read from disk cache if data, found := fc.readFromCache(cachePath); found { @@ -129,11 +125,18 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu fc.mu.Unlock() log.Debugf(ctx, "[Local Cache] cache hit: disk-read\n") fc.addTelemetryMetric("local.cache.hit") - // return data, nil // cache layer is currently no-op + return data, nil } // Get or create sync.Once for this cache key + // Check cache again under write lock to avoid race condition fc.mu.Lock() + if data, found := fc.memCache[cacheKey]; found { + fc.mu.Unlock() + log.Debugf(ctx, "[Local Cache] cache hit: in-memory (race avoided)\n") + fc.addTelemetryMetric("local.cache.hit") + return data, nil + } once, exists := fc.computeOnce[cacheKey] if !exists { once = &sync.Once{} @@ -142,24 +145,24 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu fc.mu.Unlock() // Use sync.Once to ensure only one goroutine computes the value - var result T + // Store error in a separate variable that all goroutines can access var computeErr error once.Do(func() { // Check if context is already cancelled before computing select { case <-ctx.Done(): - log.Debugf(ctx, "[Local Cache] cache miss: context is already cancelled\n") - fc.addTelemetryMetric("local.cache.miss") + log.Debugf(ctx, "[Local Cache] context cancelled before compute\n") computeErr = ctx.Err() return default: } // Compute the value - result, computeErr = compute(ctx) - if computeErr != nil { - log.Debugf(ctx, "[Local Cache] error while caching: %v\n", computeErr) + result, err := compute(ctx) + if err != nil { + log.Debugf(ctx, "[Local Cache] error while computing: %v\n", err) fc.addTelemetryMetric("local.cache.error") + computeErr = err return } @@ -169,29 +172,28 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu fc.mu.Unlock() // Async write to disk cache - log.Debugf(ctx, "[Local Cache] async writing to cache path: %s\n", cachePath) + log.Debugf(ctx, "[Local Cache] async writing to cache\n") go fc.writeToCache(cachePath, result) - log.Debugf(ctx, "[Local Cache] cache miss, but stored the compute result for future calls\n") + log.Debugf(ctx, "[Local Cache] cache miss, computed and stored result\n") fc.addTelemetryMetric("local.cache.miss") }) - // Clean up the sync.Once instance after use to prevent memory leaks - fc.mu.Lock() - delete(fc.computeOnce, cacheKey) - fc.mu.Unlock() - + // Check if computation failed if computeErr != nil { return zero, computeErr } - // If another goroutine computed the value, retrieve it from memory cache + // All goroutines retrieve the result from memCache after sync.Once completes fc.mu.RLock() - if data, found := fc.memCache[cacheKey]; found { - result = data - } + result, found := fc.memCache[cacheKey] fc.mu.RUnlock() + if !found { + // This should never happen unless there was an error + return zero, fmt.Errorf("cache inconsistency: value not found after computation") + } + return result, nil } diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index 2b98386a8f0..dc433ed8c51 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -93,17 +93,15 @@ func TestFileCacheGetOrCompute(t *testing.T) { assert.Equal(t, expectedValue, result) assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) - // Second call should return cached value + // Second call should return cached value without computing result2, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "should-not-be-called", nil }) require.NoError(t, err) - // File cache makes the second call while cache layer is no-op: - assert.Equal(t, "should-not-be-called", result2) - // assert.Equal(t, expectedValue, result2) - assert.Equal(t, int32(2), atomic.LoadInt32(&computeCalls)) + assert.Equal(t, expectedValue, result2) + assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) // Allow time for async writes to complete before test cleanup time.Sleep(50 * time.Millisecond) @@ -238,16 +236,15 @@ func TestFingerprintDeterministic(t *testing.T) { assert.Equal(t, expectedValue, result1) assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) - // Second call with fingerprint2 (should hit cache, not compute again) + // Second call with fingerprint2 (should hit cache due to deterministic hashing, not compute again) result2, err := cache.GetOrCompute(ctx, fingerprint2, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "should-not-be-called", nil }) require.NoError(t, err) - // File cache makes the second call while cache layer is no-op: - assert.Equal(t, "should-not-be-called", result2) - assert.Equal(t, int32(2), atomic.LoadInt32(&computeCalls)) // Should still be 1 + assert.Equal(t, expectedValue, result2) + assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) // Should still be 1 // Allow time for async writes to complete before test cleanup time.Sleep(50 * time.Millisecond) From 5d7805645f05f9be38b0c8561211dd5a995e512c Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 4 Nov 2025 17:15:26 +0100 Subject: [PATCH 52/87] fixes --- acceptance/cache/clear/out.test.toml | 2 +- acceptance/cache/simple/out.test.toml | 2 +- libs/cache/file_cache.go | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/acceptance/cache/clear/out.test.toml b/acceptance/cache/clear/out.test.toml index e092fd5ed6a..d560f1de043 100644 --- a/acceptance/cache/clear/out.test.toml +++ b/acceptance/cache/clear/out.test.toml @@ -2,4 +2,4 @@ Local = true Cloud = false [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct-exp"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/cache/simple/out.test.toml b/acceptance/cache/simple/out.test.toml index e092fd5ed6a..d560f1de043 100644 --- a/acceptance/cache/simple/out.test.toml +++ b/acceptance/cache/simple/out.test.toml @@ -2,4 +2,4 @@ Local = true Cloud = false [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct-exp"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 204f3b5e272..a613b8f18a9 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "encoding/hex" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -191,7 +192,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu if !found { // This should never happen unless there was an error - return zero, fmt.Errorf("cache inconsistency: value not found after computation") + return zero, errors.New("cache inconsistency: value not found after computation") } return result, nil From 127853f3e090c9df5ba712dfc82a74928bbf6a38 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 7 Nov 2025 11:52:01 +0100 Subject: [PATCH 53/87] updated output --- acceptance/cache/clear/output.txt | 10 ++++------ acceptance/cache/simple/output.txt | 6 ++---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/acceptance/cache/clear/output.txt b/acceptance/cache/clear/output.txt index efa74d102be..d9da3d80efa 100644 --- a/acceptance/cache/clear/output.txt +++ b/acceptance/cache/clear/output.txt @@ -2,17 +2,15 @@ === First call in a session is expected to be a cache miss: [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled -[DEBUG_TIMESTAMP] Debug: [Local Cache] using fingerprint with hash: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, but stored the compute result for future calls +[DEBUG_TIMESTAMP] Debug: [Local Cache] async writing to cache +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computed and stored result === Second call in a session is expected to be a cache hit [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled -[DEBUG_TIMESTAMP] Debug: [Local Cache] using fingerprint with hash: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit: disk-read -[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, but stored the compute result for future calls >>> [CLI] cache clear Cache cleared successfully from [TEST_TMP_DIR]/.cache @@ -20,6 +18,6 @@ Cache cleared successfully from [TEST_TMP_DIR]/.cache === First call after a clear is expected to be a cache miss: [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled -[DEBUG_TIMESTAMP] Debug: [Local Cache] using fingerprint with hash: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, but stored the compute result for future calls +[DEBUG_TIMESTAMP] Debug: [Local Cache] async writing to cache +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computed and stored result diff --git a/acceptance/cache/simple/output.txt b/acceptance/cache/simple/output.txt index 4272a06a49f..c02ccfb3c94 100644 --- a/acceptance/cache/simple/output.txt +++ b/acceptance/cache/simple/output.txt @@ -2,17 +2,15 @@ === First call in a session is expected to be a cache miss: [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled -[DEBUG_TIMESTAMP] Debug: [Local Cache] using fingerprint with hash: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, but stored the compute result for future calls +[DEBUG_TIMESTAMP] Debug: [Local Cache] async writing to cache +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computed and stored result === Second call in a session is expected to be a cache hit [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled -[DEBUG_TIMESTAMP] Debug: [Local Cache] using fingerprint with hash: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit: disk-read -[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, but stored the compute result for future calls === Bundle deploy should send telemetry values From cf15cd02c8482e5a7bea89184c76b053f8d96655 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 13 Nov 2025 13:22:31 +0100 Subject: [PATCH 54/87] fixed output --- .../out.requests.txt | 4 -- .../target-is-passed/default/out.requests.txt | 18 -------- .../from_flag/out.requests.txt | 9 ---- .../simple/out.requests.deploy.direct.json | 45 ++++++++----------- .../simple/out.requests.deploy.terraform.json | 43 +++++++----------- .../simple/out.requests.destroy.direct.json | 31 +++++-------- .../out.requests.destroy.terraform.json | 27 ++++------- .../simple/out.requests.plan.direct.json | 15 ++----- .../simple/out.requests.plan.terraform.json | 15 ++----- .../simple/out.requests.plan2.direct.json | 21 +++------ .../simple/out.requests.plan2.terraform.json | 19 +++----- .../simple/out.requests.run.direct.json | 15 ++----- .../simple/out.requests.run.terraform.json | 15 ++----- .../simple/out.requests.summary.direct.json | 15 ++----- .../out.requests.summary.terraform.json | 15 ++----- .../simple/out.requests.validate.direct.json | 8 ++-- .../out.requests.validate.terraform.json | 8 ++-- 17 files changed, 92 insertions(+), 231 deletions(-) diff --git a/acceptance/bundle/resource_deps/missing_ingestion_definition/out.requests.txt b/acceptance/bundle/resource_deps/missing_ingestion_definition/out.requests.txt index dadad9574ec..19e52ab2a18 100644 --- a/acceptance/bundle/resource_deps/missing_ingestion_definition/out.requests.txt +++ b/acceptance/bundle/resource_deps/missing_ingestion_definition/out.requests.txt @@ -23,10 +23,6 @@ "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files" } } -{ - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} { "method": "GET", "path": "/api/2.0/workspace/get-status", diff --git a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.requests.txt b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.requests.txt index 51bcdb4a74a..c2025facf79 100644 --- a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.requests.txt +++ b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.requests.txt @@ -1,21 +1,3 @@ -{ - "headers": { - "Authorization": [ - "Bearer [DATABRICKS_TOKEN]" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} -{ - "headers": { - "Authorization": [ - "Bearer [DATABRICKS_TOKEN]" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} { "headers": { "Authorization": [ diff --git a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.requests.txt b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.requests.txt index 14e9b1c59cb..95a6758dc8b 100644 --- a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.requests.txt +++ b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.requests.txt @@ -35,15 +35,6 @@ "path": "/oidc/v1/token", "raw_body": "grant_type=client_credentials\u0026scope=all-apis" } -{ - "headers": { - "Authorization": [ - "Bearer oauth-token" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} { "headers": { "Authorization": [ diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json index d5644f0f45c..7a393c1dc9b 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json @@ -1,16 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} -{ - "headers": { - "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -23,7 +14,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -36,7 +27,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -49,7 +40,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -62,7 +53,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -74,7 +65,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -87,7 +78,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -100,7 +91,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -113,7 +104,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -126,7 +117,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", @@ -138,7 +129,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", @@ -156,7 +147,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", @@ -181,7 +172,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", @@ -211,7 +202,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", @@ -236,7 +227,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", @@ -249,7 +240,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", @@ -261,7 +252,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", @@ -273,7 +264,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json index 63b501da2da..a5f58cf1513 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json @@ -1,16 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} -{ - "headers": { - "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -23,7 +14,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -36,7 +27,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -49,7 +40,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -62,7 +53,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -74,7 +65,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -87,7 +78,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -100,7 +91,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -113,7 +104,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -126,7 +117,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "POST", @@ -138,7 +129,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "POST", @@ -156,7 +147,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "POST", @@ -181,7 +172,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "POST", @@ -211,7 +202,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "POST", @@ -259,7 +250,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "POST", @@ -272,7 +263,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "POST", @@ -284,7 +275,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "POST", diff --git a/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json b/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json index cedab4e30b9..57f0968f60d 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json @@ -1,16 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} -{ - "headers": { - "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -19,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -32,7 +23,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -45,7 +36,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "DELETE", @@ -57,7 +48,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -70,7 +61,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -82,7 +73,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -95,7 +86,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -108,7 +99,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -117,7 +108,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", @@ -135,7 +126,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", diff --git a/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json index 243cc6984a6..15ae92f6c34 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json @@ -1,16 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} -{ - "headers": { - "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -19,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -32,7 +23,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -45,7 +36,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -58,7 +49,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -70,7 +61,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -83,7 +74,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -96,7 +87,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "POST", @@ -114,7 +105,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "POST", diff --git a/acceptance/bundle/user_agent/simple/out.requests.plan.direct.json b/acceptance/bundle/user_agent/simple/out.requests.plan.direct.json index 2e6fbad684d..e6858d92583 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.plan.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.plan.direct.json @@ -1,16 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} -{ - "headers": { - "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -23,7 +14,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -36,7 +27,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.plan.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.plan.terraform.json index bfe1c1a4727..bf2b6ab2d12 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.plan.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.plan.terraform.json @@ -1,16 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} -{ - "headers": { - "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -23,7 +14,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -36,7 +27,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.plan2.direct.json b/acceptance/bundle/user_agent/simple/out.requests.plan2.direct.json index 44b99a02761..9d3997dc416 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.plan2.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.plan2.direct.json @@ -1,16 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} -{ - "headers": { - "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -19,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -32,7 +23,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -45,7 +36,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -54,7 +45,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -67,7 +58,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.plan2.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.plan2.terraform.json index 22d685c0f6b..891e128a8d0 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.plan2.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.plan2.terraform.json @@ -1,16 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} -{ - "headers": { - "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -19,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -32,7 +23,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -45,7 +36,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -54,7 +45,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.run.direct.json b/acceptance/bundle/user_agent/simple/out.requests.run.direct.json index ea590f51c51..81ec3ed99b7 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.run.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.run.direct.json @@ -1,16 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} -{ - "headers": { - "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -19,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -32,7 +23,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.run.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.run.terraform.json index 38f4afd6fd5..aee0c742b53 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.run.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.run.terraform.json @@ -1,16 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} -{ - "headers": { - "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -19,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -32,7 +23,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.summary.direct.json b/acceptance/bundle/user_agent/simple/out.requests.summary.direct.json index bc89b2a0f94..4ccd0146fd8 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.summary.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.summary.direct.json @@ -1,16 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} -{ - "headers": { - "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -23,7 +14,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -36,7 +27,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.summary.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.summary.terraform.json index a1c7f50d66f..a96516d1282 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.summary.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.summary.terraform.json @@ -1,16 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} -{ - "headers": { - "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -23,7 +14,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -36,7 +27,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.validate.direct.json b/acceptance/bundle/user_agent/simple/out.requests.validate.direct.json index 76f04da1da2..b2038e434a3 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.validate.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.validate.direct.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -22,7 +22,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -34,7 +34,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "POST", diff --git a/acceptance/bundle/user_agent/simple/out.requests.validate.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.validate.terraform.json index 76f04da1da2..b2038e434a3 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.validate.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.validate.terraform.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -22,7 +22,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -34,7 +34,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" ] }, "method": "POST", From f4af271e0470f65dc28cf7de2419c4d9afea73b3 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 13 Nov 2025 15:22:18 +0100 Subject: [PATCH 55/87] disable cache for a couple of tests --- .../target-is-passed/default/out.requests.txt | 18 ++++ .../from_flag/out.requests.txt | 9 ++ .../run/scripts/databricks-cli/test.toml | 3 + acceptance/bundle/user_agent/output.txt | 100 +++++++++--------- .../simple/out.requests.deploy.direct.json | 9 ++ .../simple/out.requests.deploy.terraform.json | 9 ++ .../simple/out.requests.destroy.direct.json | 9 ++ .../out.requests.destroy.terraform.json | 9 ++ .../simple/out.requests.plan.direct.json | 9 ++ .../simple/out.requests.plan.terraform.json | 9 ++ .../simple/out.requests.plan2.direct.json | 9 ++ .../simple/out.requests.plan2.terraform.json | 9 ++ .../simple/out.requests.run.direct.json | 9 ++ .../simple/out.requests.run.terraform.json | 9 ++ .../simple/out.requests.summary.direct.json | 9 ++ .../out.requests.summary.terraform.json | 9 ++ acceptance/bundle/user_agent/test.toml | 3 + 17 files changed, 191 insertions(+), 50 deletions(-) diff --git a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.requests.txt b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.requests.txt index c2025facf79..51bcdb4a74a 100644 --- a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.requests.txt +++ b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.requests.txt @@ -1,3 +1,21 @@ +{ + "headers": { + "Authorization": [ + "Bearer [DATABRICKS_TOKEN]" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} +{ + "headers": { + "Authorization": [ + "Bearer [DATABRICKS_TOKEN]" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} { "headers": { "Authorization": [ diff --git a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.requests.txt b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.requests.txt index 95a6758dc8b..14e9b1c59cb 100644 --- a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.requests.txt +++ b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.requests.txt @@ -35,6 +35,15 @@ "path": "/oidc/v1/token", "raw_body": "grant_type=client_credentials\u0026scope=all-apis" } +{ + "headers": { + "Authorization": [ + "Bearer oauth-token" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} { "headers": { "Authorization": [ diff --git a/acceptance/bundle/run/scripts/databricks-cli/test.toml b/acceptance/bundle/run/scripts/databricks-cli/test.toml index 24cf889ece1..99d009ab1fe 100644 --- a/acceptance/bundle/run/scripts/databricks-cli/test.toml +++ b/acceptance/bundle/run/scripts/databricks-cli/test.toml @@ -1,6 +1,9 @@ RecordRequests = true IncludeRequestHeaders = ["Authorization"] +[Env] +DATABRICKS_CACHE_DISABLED = 'true' + # "client_id:client_secret" in base64 is Y2xpZW50X2lkOmNsaWVudF9zZWNyZXQ=, expect to # see this in Authorization header [[Repls]] diff --git a/acceptance/bundle/user_agent/output.txt b/acceptance/bundle/user_agent/output.txt index 30b72900f06..78407167eac 100644 --- a/acceptance/bundle/user_agent/output.txt +++ b/acceptance/bundle/user_agent/output.txt @@ -1,6 +1,6 @@ -MISS deploy.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat' -MISS deploy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat' -MISS deploy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat' +MISS deploy.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS deploy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS deploy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' OK deploy.direct /api/2.0/workspace/export engine/direct OK deploy.direct /api/2.0/workspace/export engine/direct OK deploy.direct /api/2.0/workspace/get-status engine/direct @@ -17,9 +17,9 @@ OK deploy.direct /api/2.0/workspace/delete engine/direct OK deploy.direct /api/2.0/workspace/delete engine/direct OK deploy.direct /api/2.0/workspace/mkdirs engine/direct OK deploy.direct /api/2.1/unity-catalog/schemas engine/direct -MISS deploy.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat' -MISS deploy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat' -MISS deploy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat' +MISS deploy.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS deploy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS deploy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' OK deploy.terraform /api/2.0/workspace/export engine/terraform OK deploy.terraform /api/2.0/workspace/export engine/terraform OK deploy.terraform /api/2.0/workspace/get-status engine/terraform @@ -37,10 +37,10 @@ OK deploy.terraform /api/2.0/workspace/delete engine/terraform OK deploy.terraform /api/2.0/workspace/mkdirs engine/terraform MISS deploy.terraform /api/2.1/unity-catalog/schemas/mycatalog.myschema 'databricks-tf-provider/1.96.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' MISS deploy.terraform /api/2.1/unity-catalog/schemas 'databricks-tf-provider/1.96.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' -MISS destroy.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat' -MISS destroy.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat' -MISS destroy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat' -MISS destroy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat' +MISS destroy.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS destroy.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS destroy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS destroy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' OK destroy.direct /api/2.1/unity-catalog/schemas/mycatalog.myschema engine/direct OK destroy.direct /api/2.0/workspace/export engine/direct OK destroy.direct /api/2.0/workspace/get-status engine/direct @@ -49,10 +49,10 @@ OK destroy.direct /api/2.0/workspace/get-status engine/direct OK destroy.direct /api/2.1/unity-catalog/schemas/mycatalog.myschema engine/direct OK destroy.direct /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/direct OK destroy.direct /api/2.0/workspace/delete engine/direct -MISS destroy.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat' -MISS destroy.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat' -MISS destroy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat' -MISS destroy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat' +MISS destroy.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS destroy.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS destroy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS destroy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' OK destroy.terraform /api/2.0/workspace/export engine/terraform OK destroy.terraform /api/2.0/workspace/get-status engine/terraform OK destroy.terraform /api/2.0/workspace/get-status engine/terraform @@ -61,49 +61,49 @@ OK destroy.terraform /api/2.0/workspace-files/import-file/Workspace/Users/[USE OK destroy.terraform /api/2.0/workspace/delete engine/terraform MISS destroy.terraform /api/2.1/unity-catalog/schemas/mycatalog.myschema 'databricks-tf-provider/1.96.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' MISS destroy.terraform /api/2.1/unity-catalog/schemas/mycatalog.myschema 'databricks-tf-provider/1.96.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' -MISS plan.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' -MISS plan.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' -MISS plan.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' +MISS plan.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS plan.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS plan.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' OK plan.direct /api/2.0/workspace/get-status engine/direct -MISS plan.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' -MISS plan.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' -MISS plan.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' +MISS plan.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS plan.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS plan.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' OK plan.terraform /api/2.0/workspace/get-status engine/terraform -MISS plan2.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' -MISS plan2.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' -MISS plan2.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' -MISS plan2.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' +MISS plan2.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS plan2.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS plan2.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS plan2.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' OK plan2.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json engine/direct OK plan2.direct /api/2.0/workspace/get-status engine/direct OK plan2.direct /api/2.1/unity-catalog/schemas/mycatalog.myschema engine/direct -MISS plan2.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' -MISS plan2.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' -MISS plan2.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' -MISS plan2.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' +MISS plan2.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS plan2.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS plan2.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS plan2.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' OK plan2.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json engine/terraform OK plan2.terraform /api/2.0/workspace/get-status engine/terraform MISS plan2.terraform /api/2.1/unity-catalog/schemas/mycatalog.myschema 'databricks-tf-provider/1.96.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' -MISS run.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat' -MISS run.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat' -MISS run.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat' -MISS run.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat' -MISS run.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat' -MISS run.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat' -MISS run.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat' -MISS run.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat' -MISS summary.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat' -MISS summary.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat' -MISS summary.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat' +MISS run.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS run.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS run.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS run.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS run.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS run.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS run.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS run.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS summary.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS summary.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS summary.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' OK summary.direct /api/2.0/preview/scim/v2/Me engine/direct -MISS summary.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat' -MISS summary.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat' -MISS summary.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat' +MISS summary.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS summary.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS summary.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' OK summary.terraform /api/2.0/preview/scim/v2/Me engine/terraform -MISS validate.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat' -MISS validate.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat' -MISS validate.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat' -MISS validate.direct /api/2.0/workspace/mkdirs 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat' -MISS validate.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat' -MISS validate.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat' -MISS validate.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat' -MISS validate.terraform /api/2.0/workspace/mkdirs 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat' +MISS validate.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS validate.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS validate.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS validate.direct /api/2.0/workspace/mkdirs 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS validate.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS validate.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS validate.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS validate.terraform /api/2.0/workspace/mkdirs 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json index 7a393c1dc9b..22767f4e21d 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json @@ -1,3 +1,12 @@ +{ + "headers": { + "User-Agent": [ + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} { "headers": { "User-Agent": [ diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json index a5f58cf1513..d2f54870eeb 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json @@ -1,3 +1,12 @@ +{ + "headers": { + "User-Agent": [ + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} { "headers": { "User-Agent": [ diff --git a/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json b/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json index 57f0968f60d..1a12922fe6b 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json @@ -1,3 +1,12 @@ +{ + "headers": { + "User-Agent": [ + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} { "headers": { "User-Agent": [ diff --git a/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json index 15ae92f6c34..3cf3bf807b2 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json @@ -1,3 +1,12 @@ +{ + "headers": { + "User-Agent": [ + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} { "headers": { "User-Agent": [ diff --git a/acceptance/bundle/user_agent/simple/out.requests.plan.direct.json b/acceptance/bundle/user_agent/simple/out.requests.plan.direct.json index e6858d92583..b20f14afd26 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.plan.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.plan.direct.json @@ -1,3 +1,12 @@ +{ + "headers": { + "User-Agent": [ + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} { "headers": { "User-Agent": [ diff --git a/acceptance/bundle/user_agent/simple/out.requests.plan.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.plan.terraform.json index bf2b6ab2d12..274856b848a 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.plan.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.plan.terraform.json @@ -1,3 +1,12 @@ +{ + "headers": { + "User-Agent": [ + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} { "headers": { "User-Agent": [ diff --git a/acceptance/bundle/user_agent/simple/out.requests.plan2.direct.json b/acceptance/bundle/user_agent/simple/out.requests.plan2.direct.json index 9d3997dc416..1f88794d074 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.plan2.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.plan2.direct.json @@ -1,3 +1,12 @@ +{ + "headers": { + "User-Agent": [ + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} { "headers": { "User-Agent": [ diff --git a/acceptance/bundle/user_agent/simple/out.requests.plan2.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.plan2.terraform.json index 891e128a8d0..5a770a3c84c 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.plan2.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.plan2.terraform.json @@ -1,3 +1,12 @@ +{ + "headers": { + "User-Agent": [ + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} { "headers": { "User-Agent": [ diff --git a/acceptance/bundle/user_agent/simple/out.requests.run.direct.json b/acceptance/bundle/user_agent/simple/out.requests.run.direct.json index 81ec3ed99b7..c545c56e0e4 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.run.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.run.direct.json @@ -1,3 +1,12 @@ +{ + "headers": { + "User-Agent": [ + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} { "headers": { "User-Agent": [ diff --git a/acceptance/bundle/user_agent/simple/out.requests.run.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.run.terraform.json index aee0c742b53..cef0fc2e0a4 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.run.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.run.terraform.json @@ -1,3 +1,12 @@ +{ + "headers": { + "User-Agent": [ + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} { "headers": { "User-Agent": [ diff --git a/acceptance/bundle/user_agent/simple/out.requests.summary.direct.json b/acceptance/bundle/user_agent/simple/out.requests.summary.direct.json index 4ccd0146fd8..34f6ce09583 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.summary.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.summary.direct.json @@ -1,3 +1,12 @@ +{ + "headers": { + "User-Agent": [ + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} { "headers": { "User-Agent": [ diff --git a/acceptance/bundle/user_agent/simple/out.requests.summary.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.summary.terraform.json index a96516d1282..d1c852db6fb 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.summary.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.summary.terraform.json @@ -1,3 +1,12 @@ +{ + "headers": { + "User-Agent": [ + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} { "headers": { "User-Agent": [ diff --git a/acceptance/bundle/user_agent/test.toml b/acceptance/bundle/user_agent/test.toml index 2bb1fd7f661..0934af860a5 100644 --- a/acceptance/bundle/user_agent/test.toml +++ b/acceptance/bundle/user_agent/test.toml @@ -1,3 +1,6 @@ RecordRequests = true Local = true IncludeRequestHeaders = ["User-Agent"] + +[Env] +DATABRICKS_CACHE_DISABLED = 'true' From 6aab004c58d1dd4d623ad0751aa0f3e15743e51e Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 13 Nov 2025 15:34:04 +0100 Subject: [PATCH 56/87] fix output --- acceptance/bundle/user_agent/output.txt | 100 +++++++++--------- .../simple/out.requests.deploy.direct.json | 38 +++---- .../simple/out.requests.deploy.terraform.json | 36 +++---- .../simple/out.requests.destroy.direct.json | 24 ++--- .../out.requests.destroy.terraform.json | 20 ++-- .../simple/out.requests.plan.direct.json | 8 +- .../simple/out.requests.plan.terraform.json | 8 +- .../simple/out.requests.plan2.direct.json | 14 +-- .../simple/out.requests.plan2.terraform.json | 12 +-- .../simple/out.requests.run.direct.json | 8 +- .../simple/out.requests.run.terraform.json | 8 +- .../simple/out.requests.summary.direct.json | 8 +- .../out.requests.summary.terraform.json | 8 +- .../simple/out.requests.validate.direct.json | 8 +- .../out.requests.validate.terraform.json | 8 +- 15 files changed, 154 insertions(+), 154 deletions(-) diff --git a/acceptance/bundle/user_agent/output.txt b/acceptance/bundle/user_agent/output.txt index 78407167eac..b31125f167d 100644 --- a/acceptance/bundle/user_agent/output.txt +++ b/acceptance/bundle/user_agent/output.txt @@ -1,6 +1,6 @@ -MISS deploy.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS deploy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS deploy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS deploy.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS deploy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS deploy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' OK deploy.direct /api/2.0/workspace/export engine/direct OK deploy.direct /api/2.0/workspace/export engine/direct OK deploy.direct /api/2.0/workspace/get-status engine/direct @@ -17,9 +17,9 @@ OK deploy.direct /api/2.0/workspace/delete engine/direct OK deploy.direct /api/2.0/workspace/delete engine/direct OK deploy.direct /api/2.0/workspace/mkdirs engine/direct OK deploy.direct /api/2.1/unity-catalog/schemas engine/direct -MISS deploy.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS deploy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS deploy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS deploy.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS deploy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS deploy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' OK deploy.terraform /api/2.0/workspace/export engine/terraform OK deploy.terraform /api/2.0/workspace/export engine/terraform OK deploy.terraform /api/2.0/workspace/get-status engine/terraform @@ -37,10 +37,10 @@ OK deploy.terraform /api/2.0/workspace/delete engine/terraform OK deploy.terraform /api/2.0/workspace/mkdirs engine/terraform MISS deploy.terraform /api/2.1/unity-catalog/schemas/mycatalog.myschema 'databricks-tf-provider/1.96.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' MISS deploy.terraform /api/2.1/unity-catalog/schemas 'databricks-tf-provider/1.96.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' -MISS destroy.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS destroy.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS destroy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS destroy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS destroy.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS destroy.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS destroy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS destroy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' OK destroy.direct /api/2.1/unity-catalog/schemas/mycatalog.myschema engine/direct OK destroy.direct /api/2.0/workspace/export engine/direct OK destroy.direct /api/2.0/workspace/get-status engine/direct @@ -49,10 +49,10 @@ OK destroy.direct /api/2.0/workspace/get-status engine/direct OK destroy.direct /api/2.1/unity-catalog/schemas/mycatalog.myschema engine/direct OK destroy.direct /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/direct OK destroy.direct /api/2.0/workspace/delete engine/direct -MISS destroy.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS destroy.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS destroy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS destroy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS destroy.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS destroy.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS destroy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS destroy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' OK destroy.terraform /api/2.0/workspace/export engine/terraform OK destroy.terraform /api/2.0/workspace/get-status engine/terraform OK destroy.terraform /api/2.0/workspace/get-status engine/terraform @@ -61,49 +61,49 @@ OK destroy.terraform /api/2.0/workspace-files/import-file/Workspace/Users/[USE OK destroy.terraform /api/2.0/workspace/delete engine/terraform MISS destroy.terraform /api/2.1/unity-catalog/schemas/mycatalog.myschema 'databricks-tf-provider/1.96.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' MISS destroy.terraform /api/2.1/unity-catalog/schemas/mycatalog.myschema 'databricks-tf-provider/1.96.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' -MISS plan.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS plan.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS plan.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS plan.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS plan.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS plan.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' OK plan.direct /api/2.0/workspace/get-status engine/direct -MISS plan.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS plan.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS plan.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS plan.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS plan.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS plan.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' OK plan.terraform /api/2.0/workspace/get-status engine/terraform -MISS plan2.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS plan2.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS plan2.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS plan2.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS plan2.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS plan2.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS plan2.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS plan2.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' OK plan2.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json engine/direct OK plan2.direct /api/2.0/workspace/get-status engine/direct OK plan2.direct /api/2.1/unity-catalog/schemas/mycatalog.myschema engine/direct -MISS plan2.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS plan2.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS plan2.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS plan2.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS plan2.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS plan2.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS plan2.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS plan2.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' OK plan2.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json engine/terraform OK plan2.terraform /api/2.0/workspace/get-status engine/terraform MISS plan2.terraform /api/2.1/unity-catalog/schemas/mycatalog.myschema 'databricks-tf-provider/1.96.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' -MISS run.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS run.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS run.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS run.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS run.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS run.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS run.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS run.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS summary.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS summary.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS summary.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS run.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS run.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS run.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS run.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS run.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS run.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS run.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS run.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS summary.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS summary.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS summary.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' OK summary.direct /api/2.0/preview/scim/v2/Me engine/direct -MISS summary.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS summary.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS summary.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS summary.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS summary.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS summary.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' OK summary.terraform /api/2.0/preview/scim/v2/Me engine/terraform -MISS validate.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS validate.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS validate.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS validate.direct /api/2.0/workspace/mkdirs 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS validate.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS validate.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS validate.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' -MISS validate.terraform /api/2.0/workspace/mkdirs 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat' +MISS validate.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS validate.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS validate.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS validate.direct /api/2.0/workspace/mkdirs 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS validate.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS validate.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS validate.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS validate.terraform /api/2.0/workspace/mkdirs 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json index 22767f4e21d..06db1e351f2 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -23,7 +23,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -36,7 +36,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -49,7 +49,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -62,7 +62,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -74,7 +74,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -87,7 +87,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -100,7 +100,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -113,7 +113,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -126,7 +126,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", @@ -138,7 +138,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", @@ -156,7 +156,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", @@ -181,7 +181,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", @@ -211,7 +211,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", @@ -236,7 +236,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", @@ -249,7 +249,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", @@ -261,7 +261,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", @@ -273,7 +273,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json index d2f54870eeb..40164869983 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -23,7 +23,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -36,7 +36,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -49,7 +49,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -62,7 +62,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -74,7 +74,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -87,7 +87,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -100,7 +100,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -113,7 +113,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -126,7 +126,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "POST", @@ -138,7 +138,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "POST", @@ -156,7 +156,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "POST", @@ -181,7 +181,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "POST", @@ -211,7 +211,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "POST", @@ -259,7 +259,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "POST", @@ -272,7 +272,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "POST", @@ -284,7 +284,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "POST", diff --git a/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json b/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json index 1a12922fe6b..1fd2ffd956f 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -19,7 +19,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -32,7 +32,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -45,7 +45,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "DELETE", @@ -57,7 +57,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -70,7 +70,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -82,7 +82,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -95,7 +95,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -108,7 +108,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -117,7 +117,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", @@ -135,7 +135,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "POST", diff --git a/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json index 3cf3bf807b2..88a0824a916 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -19,7 +19,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -32,7 +32,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -45,7 +45,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -58,7 +58,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -70,7 +70,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -83,7 +83,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -96,7 +96,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "POST", @@ -114,7 +114,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "POST", diff --git a/acceptance/bundle/user_agent/simple/out.requests.plan.direct.json b/acceptance/bundle/user_agent/simple/out.requests.plan.direct.json index b20f14afd26..24acce4f8b8 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.plan.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.plan.direct.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -23,7 +23,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -36,7 +36,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.plan.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.plan.terraform.json index 274856b848a..2f1ca3a0358 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.plan.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.plan.terraform.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -23,7 +23,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -36,7 +36,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.plan2.direct.json b/acceptance/bundle/user_agent/simple/out.requests.plan2.direct.json index 1f88794d074..1f24b211ee8 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.plan2.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.plan2.direct.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -19,7 +19,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -32,7 +32,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -45,7 +45,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -54,7 +54,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", @@ -67,7 +67,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.plan2.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.plan2.terraform.json index 5a770a3c84c..e99fa63d81f 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.plan2.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.plan2.terraform.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -19,7 +19,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -32,7 +32,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -45,7 +45,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", @@ -54,7 +54,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.run.direct.json b/acceptance/bundle/user_agent/simple/out.requests.run.direct.json index c545c56e0e4..e643533fdf3 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.run.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.run.direct.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -19,7 +19,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -32,7 +32,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.run.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.run.terraform.json index cef0fc2e0a4..ab9a92bf689 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.run.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.run.terraform.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -19,7 +19,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -32,7 +32,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.summary.direct.json b/acceptance/bundle/user_agent/simple/out.requests.summary.direct.json index 34f6ce09583..a52aefb26c4 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.summary.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.summary.direct.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -23,7 +23,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -36,7 +36,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.summary.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.summary.terraform.json index d1c852db6fb..bc9fb56a331 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.summary.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.summary.terraform.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -23,7 +23,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -36,7 +36,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.validate.direct.json b/acceptance/bundle/user_agent/simple/out.requests.validate.direct.json index b2038e434a3..f58cb7c6fc8 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.validate.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.validate.direct.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -22,7 +22,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -34,7 +34,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "POST", diff --git a/acceptance/bundle/user_agent/simple/out.requests.validate.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.validate.terraform.json index b2038e434a3..f58cb7c6fc8 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.validate.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.validate.terraform.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -22,7 +22,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "GET", @@ -34,7 +34,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] upstream/databricks-vscode-terminal upstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" ] }, "method": "POST", From 0063164ded3ab7fe5e67eb6d853a8be1b148b5f9 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 13 Nov 2025 15:49:05 +0100 Subject: [PATCH 57/87] fix output --- acceptance/bundle/user_agent/output.txt | 100 +++++++++--------- .../simple/out.requests.deploy.direct.json | 38 +++---- .../simple/out.requests.deploy.terraform.json | 36 +++---- .../simple/out.requests.destroy.direct.json | 24 ++--- .../out.requests.destroy.terraform.json | 20 ++-- .../simple/out.requests.plan.direct.json | 8 +- .../simple/out.requests.plan.terraform.json | 8 +- .../simple/out.requests.plan2.direct.json | 14 +-- .../simple/out.requests.plan2.terraform.json | 12 +-- .../simple/out.requests.run.direct.json | 8 +- .../simple/out.requests.run.terraform.json | 8 +- .../simple/out.requests.summary.direct.json | 8 +- .../out.requests.summary.terraform.json | 8 +- .../simple/out.requests.validate.direct.json | 8 +- .../out.requests.validate.terraform.json | 8 +- 15 files changed, 154 insertions(+), 154 deletions(-) diff --git a/acceptance/bundle/user_agent/output.txt b/acceptance/bundle/user_agent/output.txt index b31125f167d..30b72900f06 100644 --- a/acceptance/bundle/user_agent/output.txt +++ b/acceptance/bundle/user_agent/output.txt @@ -1,6 +1,6 @@ -MISS deploy.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS deploy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS deploy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS deploy.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat' +MISS deploy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat' +MISS deploy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat' OK deploy.direct /api/2.0/workspace/export engine/direct OK deploy.direct /api/2.0/workspace/export engine/direct OK deploy.direct /api/2.0/workspace/get-status engine/direct @@ -17,9 +17,9 @@ OK deploy.direct /api/2.0/workspace/delete engine/direct OK deploy.direct /api/2.0/workspace/delete engine/direct OK deploy.direct /api/2.0/workspace/mkdirs engine/direct OK deploy.direct /api/2.1/unity-catalog/schemas engine/direct -MISS deploy.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS deploy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS deploy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS deploy.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat' +MISS deploy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat' +MISS deploy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat' OK deploy.terraform /api/2.0/workspace/export engine/terraform OK deploy.terraform /api/2.0/workspace/export engine/terraform OK deploy.terraform /api/2.0/workspace/get-status engine/terraform @@ -37,10 +37,10 @@ OK deploy.terraform /api/2.0/workspace/delete engine/terraform OK deploy.terraform /api/2.0/workspace/mkdirs engine/terraform MISS deploy.terraform /api/2.1/unity-catalog/schemas/mycatalog.myschema 'databricks-tf-provider/1.96.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' MISS deploy.terraform /api/2.1/unity-catalog/schemas 'databricks-tf-provider/1.96.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' -MISS destroy.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS destroy.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS destroy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS destroy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS destroy.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat' +MISS destroy.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat' +MISS destroy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat' +MISS destroy.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat' OK destroy.direct /api/2.1/unity-catalog/schemas/mycatalog.myschema engine/direct OK destroy.direct /api/2.0/workspace/export engine/direct OK destroy.direct /api/2.0/workspace/get-status engine/direct @@ -49,10 +49,10 @@ OK destroy.direct /api/2.0/workspace/get-status engine/direct OK destroy.direct /api/2.1/unity-catalog/schemas/mycatalog.myschema engine/direct OK destroy.direct /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/direct OK destroy.direct /api/2.0/workspace/delete engine/direct -MISS destroy.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS destroy.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS destroy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS destroy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS destroy.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat' +MISS destroy.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat' +MISS destroy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat' +MISS destroy.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat' OK destroy.terraform /api/2.0/workspace/export engine/terraform OK destroy.terraform /api/2.0/workspace/get-status engine/terraform OK destroy.terraform /api/2.0/workspace/get-status engine/terraform @@ -61,49 +61,49 @@ OK destroy.terraform /api/2.0/workspace-files/import-file/Workspace/Users/[USE OK destroy.terraform /api/2.0/workspace/delete engine/terraform MISS destroy.terraform /api/2.1/unity-catalog/schemas/mycatalog.myschema 'databricks-tf-provider/1.96.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' MISS destroy.terraform /api/2.1/unity-catalog/schemas/mycatalog.myschema 'databricks-tf-provider/1.96.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' -MISS plan.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS plan.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS plan.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS plan.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' +MISS plan.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' +MISS plan.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' OK plan.direct /api/2.0/workspace/get-status engine/direct -MISS plan.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS plan.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS plan.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS plan.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' +MISS plan.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' +MISS plan.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' OK plan.terraform /api/2.0/workspace/get-status engine/terraform -MISS plan2.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS plan2.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS plan2.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS plan2.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS plan2.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' +MISS plan2.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' +MISS plan2.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' +MISS plan2.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' OK plan2.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json engine/direct OK plan2.direct /api/2.0/workspace/get-status engine/direct OK plan2.direct /api/2.1/unity-catalog/schemas/mycatalog.myschema engine/direct -MISS plan2.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS plan2.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS plan2.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS plan2.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS plan2.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' +MISS plan2.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' +MISS plan2.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' +MISS plan2.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat' OK plan2.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json engine/terraform OK plan2.terraform /api/2.0/workspace/get-status engine/terraform MISS plan2.terraform /api/2.1/unity-catalog/schemas/mycatalog.myschema 'databricks-tf-provider/1.96.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' -MISS run.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS run.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS run.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS run.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS run.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS run.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS run.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS run.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS summary.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS summary.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS summary.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS run.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat' +MISS run.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat' +MISS run.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat' +MISS run.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat' +MISS run.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat' +MISS run.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat' +MISS run.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat' +MISS run.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat' +MISS summary.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat' +MISS summary.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat' +MISS summary.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat' OK summary.direct /api/2.0/preview/scim/v2/Me engine/direct -MISS summary.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS summary.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS summary.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS summary.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat' +MISS summary.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat' +MISS summary.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat' OK summary.terraform /api/2.0/preview/scim/v2/Me engine/terraform -MISS validate.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS validate.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS validate.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS validate.direct /api/2.0/workspace/mkdirs 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS validate.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS validate.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS validate.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' -MISS validate.terraform /api/2.0/workspace/mkdirs 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat' +MISS validate.direct /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat' +MISS validate.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat' +MISS validate.direct /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat' +MISS validate.direct /api/2.0/workspace/mkdirs 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat' +MISS validate.terraform /api/2.0/preview/scim/v2/Me 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat' +MISS validate.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat' +MISS validate.terraform /api/2.0/workspace/get-status 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat' +MISS validate.terraform /api/2.0/workspace/mkdirs 'cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat' diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json index 06db1e351f2..d5644f0f45c 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -23,7 +23,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -36,7 +36,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "GET", @@ -49,7 +49,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "GET", @@ -62,7 +62,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "GET", @@ -74,7 +74,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "GET", @@ -87,7 +87,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "GET", @@ -100,7 +100,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "GET", @@ -113,7 +113,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "GET", @@ -126,7 +126,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "POST", @@ -138,7 +138,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "POST", @@ -156,7 +156,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "POST", @@ -181,7 +181,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "POST", @@ -211,7 +211,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "POST", @@ -236,7 +236,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "POST", @@ -249,7 +249,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "POST", @@ -261,7 +261,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "POST", @@ -273,7 +273,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "POST", diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json index 40164869983..63b501da2da 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -23,7 +23,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -36,7 +36,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "GET", @@ -49,7 +49,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "GET", @@ -62,7 +62,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "GET", @@ -74,7 +74,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "GET", @@ -87,7 +87,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "GET", @@ -100,7 +100,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "GET", @@ -113,7 +113,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "GET", @@ -126,7 +126,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "POST", @@ -138,7 +138,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "POST", @@ -156,7 +156,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "POST", @@ -181,7 +181,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "POST", @@ -211,7 +211,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "POST", @@ -259,7 +259,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "POST", @@ -272,7 +272,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "POST", @@ -284,7 +284,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "POST", diff --git a/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json b/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json index 1fd2ffd956f..cedab4e30b9 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -19,7 +19,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -32,7 +32,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -45,7 +45,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "DELETE", @@ -57,7 +57,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "GET", @@ -70,7 +70,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "GET", @@ -82,7 +82,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "GET", @@ -95,7 +95,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "GET", @@ -108,7 +108,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "GET", @@ -117,7 +117,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "POST", @@ -135,7 +135,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "POST", diff --git a/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json index 88a0824a916..243cc6984a6 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -19,7 +19,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -32,7 +32,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -45,7 +45,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "GET", @@ -58,7 +58,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "GET", @@ -70,7 +70,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "GET", @@ -83,7 +83,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "GET", @@ -96,7 +96,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "POST", @@ -114,7 +114,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "POST", diff --git a/acceptance/bundle/user_agent/simple/out.requests.plan.direct.json b/acceptance/bundle/user_agent/simple/out.requests.plan.direct.json index 24acce4f8b8..2e6fbad684d 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.plan.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.plan.direct.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -23,7 +23,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -36,7 +36,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.plan.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.plan.terraform.json index 2f1ca3a0358..bfe1c1a4727 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.plan.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.plan.terraform.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -23,7 +23,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -36,7 +36,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.plan2.direct.json b/acceptance/bundle/user_agent/simple/out.requests.plan2.direct.json index 1f24b211ee8..44b99a02761 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.plan2.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.plan2.direct.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -19,7 +19,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -32,7 +32,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -45,7 +45,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "GET", @@ -54,7 +54,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "GET", @@ -67,7 +67,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.plan2.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.plan2.terraform.json index e99fa63d81f..22d685c0f6b 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.plan2.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.plan2.terraform.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -19,7 +19,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -32,7 +32,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -45,7 +45,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "GET", @@ -54,7 +54,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.run.direct.json b/acceptance/bundle/user_agent/simple/out.requests.run.direct.json index e643533fdf3..ea590f51c51 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.run.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.run.direct.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -19,7 +19,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -32,7 +32,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.run.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.run.terraform.json index ab9a92bf689..38f4afd6fd5 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.run.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.run.terraform.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -19,7 +19,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -32,7 +32,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.summary.direct.json b/acceptance/bundle/user_agent/simple/out.requests.summary.direct.json index a52aefb26c4..bc89b2a0f94 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.summary.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.summary.direct.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -23,7 +23,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -36,7 +36,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/direct auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.summary.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.summary.terraform.json index bc9fb56a331..a1c7f50d66f 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.summary.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.summary.terraform.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -23,7 +23,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -36,7 +36,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 engine/terraform auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_summary cmd-exec-id/[UUID] engine/terraform auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.validate.direct.json b/acceptance/bundle/user_agent/simple/out.requests.validate.direct.json index f58cb7c6fc8..76f04da1da2 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.validate.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.validate.direct.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -22,7 +22,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -34,7 +34,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat" ] }, "method": "POST", diff --git a/acceptance/bundle/user_agent/simple/out.requests.validate.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.validate.terraform.json index f58cb7c6fc8..76f04da1da2 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.validate.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.validate.terraform.json @@ -1,7 +1,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -10,7 +10,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -22,7 +22,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat" ] }, "method": "GET", @@ -34,7 +34,7 @@ { "headers": { "User-Agent": [ - "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] engine/direct auth/patupstream-version/2.10.3 auth/pat" + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_validate cmd-exec-id/[UUID] auth/pat" ] }, "method": "POST", From 4a1392b6ebafbb9579c594893a7124aefd71b543 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 13 Nov 2025 16:20:18 +0100 Subject: [PATCH 58/87] write cache synchornously --- libs/cache/file_cache.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index a613b8f18a9..94997274e6b 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -172,9 +172,9 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu fc.memCache[cacheKey] = result fc.mu.Unlock() - // Async write to disk cache - log.Debugf(ctx, "[Local Cache] async writing to cache\n") - go fc.writeToCache(cachePath, result) + // Write to disk cache synchronously to ensure it persists before process exits + log.Debugf(ctx, "[Local Cache] writing to cache\n") + fc.writeToCache(cachePath, result) log.Debugf(ctx, "[Local Cache] cache miss, computed and stored result\n") fc.addTelemetryMetric("local.cache.miss") From 48fd2d1afd67845f342334b9709a7b1ab96dd29c Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 13 Nov 2025 16:28:53 +0100 Subject: [PATCH 59/87] fix output --- acceptance/cache/clear/output.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/cache/clear/output.txt b/acceptance/cache/clear/output.txt index d9da3d80efa..c07070ef338 100644 --- a/acceptance/cache/clear/output.txt +++ b/acceptance/cache/clear/output.txt @@ -3,7 +3,7 @@ [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] async writing to cache +[DEBUG_TIMESTAMP] Debug: [Local Cache] writing to cache [DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computed and stored result === Second call in a session is expected to be a cache hit @@ -19,5 +19,5 @@ Cache cleared successfully from [TEST_TMP_DIR]/.cache [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] async writing to cache +[DEBUG_TIMESTAMP] Debug: [Local Cache] writing to cache [DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computed and stored result From eddf974b780b0b4322a8d7cd185764adf8dbfa0d Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 13 Nov 2025 16:44:00 +0100 Subject: [PATCH 60/87] fix cache test --- acceptance/cache/simple/output.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/cache/simple/output.txt b/acceptance/cache/simple/output.txt index c02ccfb3c94..a6d124c2a60 100644 --- a/acceptance/cache/simple/output.txt +++ b/acceptance/cache/simple/output.txt @@ -3,7 +3,7 @@ [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] async writing to cache +[DEBUG_TIMESTAMP] Debug: [Local Cache] writing to cache [DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computed and stored result === Second call in a session is expected to be a cache hit From ef265180bc366e7eb7a57754bfd7b94e7b0795cb Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 17 Nov 2025 11:37:31 +0100 Subject: [PATCH 61/87] removed async logic + addressed feedback --- acceptance/cache/clear/output.txt | 10 +- acceptance/cache/clear/script | 4 +- acceptance/cache/simple/output.txt | 6 +- acceptance/cache/simple/script | 4 +- libs/cache/cache.go | 54 +---- libs/cache/cleanup.go | 200 ---------------- libs/cache/cleanup_test.go | 328 --------------------------- libs/cache/file_cache.go | 216 +++++++----------- libs/cache/file_cache_expiry_test.go | 39 +--- libs/cache/file_cache_test.go | 81 ++++--- 10 files changed, 159 insertions(+), 783 deletions(-) delete mode 100644 libs/cache/cleanup.go delete mode 100644 libs/cache/cleanup_test.go diff --git a/acceptance/cache/clear/output.txt b/acceptance/cache/clear/output.txt index c07070ef338..4ee4c6e8372 100644 --- a/acceptance/cache/clear/output.txt +++ b/acceptance/cache/clear/output.txt @@ -3,14 +3,14 @@ [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] writing to cache -[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computed and stored result +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing +[DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result === Second call in a session is expected to be a cache hit [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit: disk-read +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit >>> [CLI] cache clear Cache cleared successfully from [TEST_TMP_DIR]/.cache @@ -19,5 +19,5 @@ Cache cleared successfully from [TEST_TMP_DIR]/.cache [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] writing to cache -[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computed and stored result +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing +[DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result diff --git a/acceptance/cache/clear/script b/acceptance/cache/clear/script index 076fbd722b4..7c082ae82dd 100644 --- a/acceptance/cache/clear/script +++ b/acceptance/cache/clear/script @@ -1,4 +1,4 @@ -export DATABRICKS_CACHE_FOLDER=$(pwd)/.cache +export DATABRICKS_CACHE_DIR=$(pwd)/.cache $CLI cache clear &> /dev/null @@ -13,4 +13,4 @@ trace $CLI cache clear title "First call after a clear is expected to be a cache miss:\n" trace $CLI bundle validate --debug 2>&1 | grep "Local Cache" | grep -v "cache path" -rm -rf "${DATABRICKS_CACHE_FOLDER}" +rm -rf "${DATABRICKS_CACHE_DIR}" diff --git a/acceptance/cache/simple/output.txt b/acceptance/cache/simple/output.txt index a6d124c2a60..d8810bbf73b 100644 --- a/acceptance/cache/simple/output.txt +++ b/acceptance/cache/simple/output.txt @@ -3,14 +3,14 @@ [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] writing to cache -[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computed and stored result +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing +[DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result === Second call in a session is expected to be a cache hit [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit: disk-read +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit === Bundle deploy should send telemetry values diff --git a/acceptance/cache/simple/script b/acceptance/cache/simple/script index 6a1f1de5a6e..a2907174bf3 100644 --- a/acceptance/cache/simple/script +++ b/acceptance/cache/simple/script @@ -1,4 +1,4 @@ -export DATABRICKS_CACHE_FOLDER=$(pwd)/.cache +export DATABRICKS_CACHE_DIR=$(pwd)/.cache $CLI cache clear &> /dev/null @@ -13,4 +13,4 @@ trace $CLI bundle deploy -p dogfood trace print_telemetry_bool_values | grep "local.cache" rm out.requests.txt -rm -rf "${DATABRICKS_CACHE_FOLDER}" +rm -rf "${DATABRICKS_CACHE_DIR}" diff --git a/libs/cache/cache.go b/libs/cache/cache.go index d2931a270ce..82299b83a0f 100644 --- a/libs/cache/cache.go +++ b/libs/cache/cache.go @@ -6,75 +6,45 @@ import ( "encoding/hex" "encoding/json" "fmt" - "sort" ) // Cache provides an abstract interface for caching content to local disk. // Implementations should handle storing and retrieving cached components // using fingerprints for cache invalidation. +// Cache operations fail open: if caching fails, the compute function is still called. type Cache[T any] interface { // GetOrCompute retrieves cached content for the given fingerprint, or computes it using the provided function. // If the content is found in cache, it is returned directly. // If not found, the compute function is called, its result is cached, and then returned. // The fingerprint can be any struct that will be serialized deterministically for cache key generation. - // Returns an error if the cache operation or compute function fails. + // Cache failures do not block computation - if caching fails, compute is called anyway. + // Returns an error only if the compute function fails. GetOrCompute(ctx context.Context, fingerprint any, compute func(ctx context.Context) (T, error)) (T, error) } // fingerprintToHash converts any struct to a deterministic string representation for use as a cache key. +// For structs, json.Marshal uses struct field order, not JSON tag order. To ensure deterministic +// hashing regardless of struct field order, we convert to a map which json.Marshal sorts by key. func fingerprintToHash(fingerprint any) (string, error) { - // Serialize to JSON with sorted keys for deterministic output + // Marshal to JSON data, err := json.Marshal(fingerprint) if err != nil { return "", fmt.Errorf("failed to marshal fingerprint: %w", err) } - // Parse back to ensure consistent key ordering - var obj any - if err := json.Unmarshal(data, &obj); err != nil { + // Unmarshal to map to ensure key ordering + var m map[string]any + if err := json.Unmarshal(data, &m); err != nil { return "", fmt.Errorf("failed to unmarshal fingerprint: %w", err) } - // Sort keys deterministically - normalized := normalizeForFingerprint(obj) - - // Re-marshal with normalized structure - normalizedData, err := json.Marshal(normalized) + // Marshal map (map keys are sorted by json.Marshal) + normalizedData, err := json.Marshal(m) if err != nil { return "", fmt.Errorf("failed to marshal normalized fingerprint: %w", err) } - // Hash the result for a consistent, reasonably-sized key + // Hash for consistent, reasonably-sized key hash := sha256.Sum256(normalizedData) return hex.EncodeToString(hash[:]), nil } - -// normalizeForFingerprint recursively sorts map keys to ensure deterministic serialization. -func normalizeForFingerprint(obj any) any { - switch v := obj.(type) { - case map[string]any: - // Sort keys - keys := make([]string, 0, len(v)) - for k := range v { - keys = append(keys, k) - } - sort.Strings(keys) - - // Create ordered map - result := make(map[string]any, len(v)) - for _, k := range keys { - result[k] = normalizeForFingerprint(v[k]) - } - return result - case []any: - // Normalize each element in the slice - result := make([]any, len(v)) - for i, item := range v { - result[i] = normalizeForFingerprint(item) - } - return result - default: - // Primitive types are returned as-is - return v - } -} diff --git a/libs/cache/cleanup.go b/libs/cache/cleanup.go deleted file mode 100644 index d0dcede0499..00000000000 --- a/libs/cache/cleanup.go +++ /dev/null @@ -1,200 +0,0 @@ -package cache - -import ( - "context" - "encoding/json" - "os" - "path/filepath" - "strings" - "sync" - "time" - - "github.com/databricks/cli/libs/log" -) - -// CleanupConfig holds configuration for cache cleanup. -type CleanupConfig struct { - MaxAge time.Duration // Maximum age of cache files before cleanup - DryRun bool // If true, only logs what would be deleted -} - -// DefaultCleanupConfig returns sensible defaults for cache cleanup. -func DefaultCleanupConfig() CleanupConfig { - return CleanupConfig{ - MaxAge: 7 * 24 * time.Hour, // 7 days - DryRun: false, - } -} - -// CleanupManager manages background cleanup of cache files. -type CleanupManager struct { - config CleanupConfig - stopCh chan struct{} - stoppedCh chan struct{} - mu sync.Mutex - running bool - stopped bool -} - -// NewCleanupManager creates a new cleanup manager with the given configuration. -func NewCleanupManager(config CleanupConfig) *CleanupManager { - return &CleanupManager{ - config: config, - stopCh: make(chan struct{}), - stoppedCh: make(chan struct{}), - } -} - -// Start runs a one-time cleanup of cache files. -// This is non-blocking and will not prevent the main process from exiting. -func (cm *CleanupManager) Start(ctx context.Context, cacheDir string) { - cm.mu.Lock() - defer cm.mu.Unlock() - - if cm.running || cm.stopped { - return // Already running or stopped - } - - cm.running = true - - go func() { - defer func() { - cm.mu.Lock() - cm.running = false - cm.mu.Unlock() - close(cm.stoppedCh) - }() - - log.Debugf(ctx, "[Cache Cleanup] Starting cleanup manager for directory: %s", cacheDir) - - // Perform one-time cleanup - cm.cleanup(ctx, cacheDir) - - log.Debugf(ctx, "[Cache Cleanup] Cleanup manager finished") - }() -} - -// Stop gracefully stops the cleanup manager. -// This is non-blocking and returns immediately. -func (cm *CleanupManager) Stop() { - cm.mu.Lock() - defer cm.mu.Unlock() - - if !cm.running || cm.stopped { - return - } - - cm.stopped = true - close(cm.stopCh) -} - -// Wait waits for the cleanup manager to stop completely. -// This should only be used in tests or shutdown scenarios where you need to wait. -func (cm *CleanupManager) Wait() { - <-cm.stoppedCh -} - -// cleanup performs the actual cleanup of old cache files. -func (cm *CleanupManager) cleanup(ctx context.Context, cacheDir string) { - log.Debugf(ctx, "[Cache Cleanup] Starting cleanup scan of directory: %s", cacheDir) - - // Check if cache directory exists - if _, err := os.Stat(cacheDir); os.IsNotExist(err) { - log.Debugf(ctx, "[Cache Cleanup] Cache directory does not exist: %s", cacheDir) - return - } - - var deletedCount, scannedCount int - var totalSize, deletedSize int64 - now := time.Now() - - err := filepath.Walk(cacheDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - log.Debugf(ctx, "[Cache Cleanup] Error accessing path %s: %v", path, err) - return nil // Continue with other files - } - - // Skip directories and non-cache files - if info.IsDir() || !strings.HasSuffix(info.Name(), ".json") { - return nil - } - - scannedCount++ - totalSize += info.Size() - - shouldDelete, fileAge := cm.shouldDeleteFile(path, now) - if shouldDelete { - deletedSize += info.Size() - deletedCount++ - - if cm.config.DryRun { - log.Debugf(ctx, "[Cache Cleanup] Would delete old cache file: %s (age: %v)", path, fileAge) - } else { - if err := os.Remove(path); err != nil { - log.Debugf(ctx, "[Cache Cleanup] Failed to delete cache file %s: %v", path, err) - } else { - log.Debugf(ctx, "[Cache Cleanup] Deleted old cache file: %s (age: %v)", path, fileAge) - } - } - } - - return nil - }) - if err != nil { - log.Debugf(ctx, "[Cache Cleanup] Error during cleanup scan: %v", err) - } - - action := "deleted" - if cm.config.DryRun { - action = "would delete" - } - - log.Debugf(ctx, "[Cache Cleanup] Cleanup complete: scanned %d files (%.2f MB), %s %d files (%.2f MB)", - scannedCount, float64(totalSize)/(1024*1024), - action, deletedCount, float64(deletedSize)/(1024*1024)) -} - -// shouldDeleteFile determines if a cache file should be deleted based on its expiry. -func (cm *CleanupManager) shouldDeleteFile(path string, now time.Time) (bool, time.Duration) { - // Try to read the cache entry to get the expiry - data, err := os.ReadFile(path) - if err != nil { - // If we can't read the file, use file modification time as fallback - if info, statErr := os.Stat(path); statErr == nil { - age := time.Since(info.ModTime()) - // Use MaxAge fallback for files without expiry information - return info.ModTime().Add(cm.config.MaxAge).Before(now), age - } - return true, time.Duration(0) // Delete unreadable files - } - - var entry cacheEntry - if err := json.Unmarshal(data, &entry); err != nil { - // If we can't parse the cache entry, use file modification time as fallback - if info, statErr := os.Stat(path); statErr == nil { - age := time.Since(info.ModTime()) - // Use MaxAge fallback for files without expiry information - return info.ModTime().Add(cm.config.MaxAge).Before(now), age - } - return true, time.Duration(0) // Delete unparseable files - } - - // Check if the file has expired - if !entry.Expiry.IsZero() { - isExpired := entry.Expiry.Before(now) - age := now.Sub(entry.Expiry) - if age < 0 { - age = 0 // File hasn't expired yet - } - return isExpired, age - } - - // Fallback to Timestamp field for backward compatibility - if !entry.Timestamp.IsZero() { - age := time.Since(entry.Timestamp) - return entry.Timestamp.Add(cm.config.MaxAge).Before(now), age - } - - // If neither expiry nor timestamp is available, delete the file - return true, time.Duration(0) -} diff --git a/libs/cache/cleanup_test.go b/libs/cache/cleanup_test.go deleted file mode 100644 index 91e1df1c58d..00000000000 --- a/libs/cache/cleanup_test.go +++ /dev/null @@ -1,328 +0,0 @@ -package cache - -import ( - "context" - "encoding/json" - "os" - "path/filepath" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestDefaultCleanupConfig(t *testing.T) { - config := DefaultCleanupConfig() - assert.Equal(t, 7*24*time.Hour, config.MaxAge) - assert.False(t, config.DryRun) -} - -func TestCleanupManager_Start_Stop(t *testing.T) { - ctx := context.Background() - tempDir := t.TempDir() - - config := CleanupConfig{ - MaxAge: time.Hour, - DryRun: false, - } - - manager := NewCleanupManager(config) - - // Start the manager - manager.Start(ctx, tempDir) - assert.True(t, manager.running) - - // Stop the manager - manager.Stop() - - // Wait for it to stop with timeout - done := make(chan struct{}) - go func() { - manager.Wait() - close(done) - }() - - select { - case <-done: - assert.False(t, manager.running) - case <-time.After(5 * time.Second): - t.Fatal("Cleanup manager did not stop within timeout") - } -} - -func TestCleanupManager_CleanupOldFiles(t *testing.T) { - ctx := context.Background() - tempDir := t.TempDir() - - config := CleanupConfig{ - MaxAge: time.Hour, - DryRun: false, - } - - manager := NewCleanupManager(config) - - // Create test files with different ages - now := time.Now() - - // Create an old file (should be deleted) - oldFile := filepath.Join(tempDir, "old_file.json") - oldEntry := cacheEntry{ - Data: json.RawMessage(`"old_data"`), - Timestamp: now.Add(-2 * time.Hour), // 2 hours old - } - oldData, err := json.Marshal(oldEntry) - require.NoError(t, err) - require.NoError(t, os.WriteFile(oldFile, oldData, 0o644)) - - // Create a recent file (should not be deleted) - recentFile := filepath.Join(tempDir, "recent_file.json") - recentEntry := cacheEntry{ - Data: json.RawMessage(`"recent_data"`), - Timestamp: now.Add(-30 * time.Minute), // 30 minutes old - } - recentData, err := json.Marshal(recentEntry) - require.NoError(t, err) - require.NoError(t, os.WriteFile(recentFile, recentData, 0o644)) - - // Create a non-cache file (should be ignored) - nonCacheFile := filepath.Join(tempDir, "not_cache.txt") - require.NoError(t, os.WriteFile(nonCacheFile, []byte("not cache"), 0o644)) - - // Run cleanup manually - manager.cleanup(ctx, tempDir) - - // Check results - _, err = os.Stat(oldFile) - assert.True(t, os.IsNotExist(err), "Old file should be deleted") - - _, err = os.Stat(recentFile) - assert.False(t, os.IsNotExist(err), "Recent file should not be deleted") - - _, err = os.Stat(nonCacheFile) - assert.False(t, os.IsNotExist(err), "Non-cache file should not be deleted") -} - -func TestCleanupManager_DryRun(t *testing.T) { - ctx := context.Background() - tempDir := t.TempDir() - - config := CleanupConfig{ - MaxAge: time.Hour, - DryRun: true, // Dry run mode - } - - manager := NewCleanupManager(config) - - // Create an old file - now := time.Now() - oldFile := filepath.Join(tempDir, "old_file.json") - oldEntry := cacheEntry{ - Data: json.RawMessage(`"old_data"`), - Timestamp: now.Add(-2 * time.Hour), - } - oldData, err := json.Marshal(oldEntry) - require.NoError(t, err) - require.NoError(t, os.WriteFile(oldFile, oldData, 0o644)) - - // Run cleanup in dry run mode - manager.cleanup(ctx, tempDir) - - // File should still exist in dry run mode - _, err = os.Stat(oldFile) - assert.False(t, os.IsNotExist(err), "File should not be deleted in dry run mode") -} - -func TestCleanupManager_CorruptedFiles(t *testing.T) { - ctx := context.Background() - tempDir := t.TempDir() - - config := CleanupConfig{ - MaxAge: time.Hour, - DryRun: false, - } - - manager := NewCleanupManager(config) - - // Create a corrupted cache file (invalid JSON) - corruptedFile := filepath.Join(tempDir, "corrupted.json") - require.NoError(t, os.WriteFile(corruptedFile, []byte("invalid json"), 0o644)) - - // Set old modification time - oldTime := time.Now().Add(-2 * time.Hour) - require.NoError(t, os.Chtimes(corruptedFile, oldTime, oldTime)) - - // Create a file with invalid cache entry structure - invalidStructureFile := filepath.Join(tempDir, "invalid_structure.json") - require.NoError(t, os.WriteFile(invalidStructureFile, []byte(`{"invalid": "structure"}`), 0o644)) - - // Set old modification time - require.NoError(t, os.Chtimes(invalidStructureFile, oldTime, oldTime)) - - // Run cleanup - corrupted files should be deleted - manager.cleanup(ctx, tempDir) - - // Both corrupted files should be deleted - _, err := os.Stat(corruptedFile) - assert.True(t, os.IsNotExist(err), "Corrupted file should be deleted") - - _, err = os.Stat(invalidStructureFile) - assert.True(t, os.IsNotExist(err), "Invalid structure file should be deleted") -} - -func TestCleanupManager_NonexistentDirectory(t *testing.T) { - ctx := context.Background() - nonexistentDir := "/nonexistent/directory" - - config := CleanupConfig{ - MaxAge: time.Hour, - DryRun: false, - } - - manager := NewCleanupManager(config) - - // This should not panic or error when directory doesn't exist - manager.cleanup(ctx, nonexistentDir) -} - -func TestShouldDeleteFile(t *testing.T) { - tempDir := t.TempDir() - - manager := NewCleanupManager(DefaultCleanupConfig()) - now := time.Now() - - // Test with valid cache entry with expiry - expired file - expiredFile := filepath.Join(tempDir, "expired.json") - expiredEntry := cacheEntry{ - Data: json.RawMessage(`"data"`), - Expiry: now.Add(-time.Hour), // Expired 1 hour ago - } - expiredData, err := json.Marshal(expiredEntry) - require.NoError(t, err) - require.NoError(t, os.WriteFile(expiredFile, expiredData, 0o644)) - - shouldDelete, age := manager.shouldDeleteFile(expiredFile, now) - assert.True(t, shouldDelete, "Expired file should be marked for deletion") - assert.GreaterOrEqual(t, age, time.Hour, "Age should reflect time since expiry") - - // Test with valid cache entry with expiry - not expired file - validFile := filepath.Join(tempDir, "valid.json") - validEntry := cacheEntry{ - Data: json.RawMessage(`"data"`), - Expiry: now.Add(time.Hour), // Expires in 1 hour - } - validData, err := json.Marshal(validEntry) - require.NoError(t, err) - require.NoError(t, os.WriteFile(validFile, validData, 0o644)) - - shouldDelete, age = manager.shouldDeleteFile(validFile, now) - assert.False(t, shouldDelete, "Valid file should not be marked for deletion") - assert.Equal(t, time.Duration(0), age, "Age should be 0 for unexpired files") - - // Test with legacy timestamp field (backward compatibility) - legacyFile := filepath.Join(tempDir, "legacy.json") - legacyEntry := cacheEntry{ - Data: json.RawMessage(`"data"`), - Timestamp: now.Add(-2 * time.Hour), // Created 2 hours ago - } - legacyData, err := json.Marshal(legacyEntry) - require.NoError(t, err) - require.NoError(t, os.WriteFile(legacyFile, legacyData, 0o644)) - - shouldDelete, age = manager.shouldDeleteFile(legacyFile, now) - // Should not be deleted since MaxAge is 7 days by default, but 2 hours < 7 days - assert.False(t, shouldDelete, "Legacy file should not be deleted if within MaxAge") - assert.Greater(t, age, 2*time.Hour, "Age should be based on timestamp") - - // Test with invalid JSON - should use file modification time - invalidFile := filepath.Join(tempDir, "invalid.json") - require.NoError(t, os.WriteFile(invalidFile, []byte("invalid"), 0o644)) - // Set modification time to be old - oldTime := now.Add(-8 * 24 * time.Hour) // 8 days ago (beyond default MaxAge) - require.NoError(t, os.Chtimes(invalidFile, oldTime, oldTime)) - - shouldDelete, age = manager.shouldDeleteFile(invalidFile, now) - assert.True(t, shouldDelete, "Invalid file should be marked for deletion based on mod time") - assert.Greater(t, age, 7*24*time.Hour, "Age should be based on modification time") -} - -func TestCleanupIntegrationWithFileCache(t *testing.T) { - tempDir := t.TempDir() - - // Create file cache which should start cleanup automatically - cache, err := newFileCacheWithBaseDir[string](tempDir, 60) // 1 hour for tests - require.NoError(t, err) - require.NotNil(t, cache.cleanupMgr) - - // Stop cleanup to prevent interference with test - cache.StopCleanup() - cache.cleanupMgr.Wait() - - // Verify cache directory was created - _, err = os.Stat(tempDir) - assert.False(t, os.IsNotExist(err), "Cache directory should exist") -} - -func TestCleanupManager_MultipleStartStop(t *testing.T) { - ctx := context.Background() - tempDir := t.TempDir() - - manager := NewCleanupManager(DefaultCleanupConfig()) - - // Start multiple times - should only start once - manager.Start(ctx, tempDir) - manager.Start(ctx, tempDir) // Second start should be ignored - assert.True(t, manager.running) - - // Stop multiple times - should be safe - manager.Stop() - manager.Stop() // Second stop should be safe - - manager.Wait() - assert.False(t, manager.running) -} - -func TestCleanupFileWalk(t *testing.T) { - ctx := context.Background() - tempDir := t.TempDir() - - config := CleanupConfig{ - MaxAge: time.Hour, - DryRun: false, - } - - manager := NewCleanupManager(config) - - // Create nested directory structure - subDir := filepath.Join(tempDir, "subdir") - require.NoError(t, os.MkdirAll(subDir, 0o755)) - - now := time.Now() - - // Create old files in both root and subdirectory - oldFile1 := filepath.Join(tempDir, "old1.json") - oldFile2 := filepath.Join(subDir, "old2.json") - - for _, file := range []string{oldFile1, oldFile2} { - oldEntry := cacheEntry{ - Data: json.RawMessage(`"old_data"`), - Timestamp: now.Add(-2 * time.Hour), - } - data, err := json.Marshal(oldEntry) - require.NoError(t, err) - require.NoError(t, os.WriteFile(file, data, 0o644)) - } - - // Run cleanup - manager.cleanup(ctx, tempDir) - - // Both files should be deleted - for _, file := range []string{oldFile1, oldFile2} { - _, err := os.Stat(file) - assert.True(t, os.IsNotExist(err), "File %s should be deleted", file) - } - - // Subdirectory should still exist - _, err := os.Stat(subDir) - assert.False(t, os.IsNotExist(err), "Subdirectory should still exist") -} diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 94997274e6b..431f233d39c 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -2,57 +2,87 @@ package cache import ( "context" - "crypto/rand" - "encoding/hex" "encoding/json" - "errors" "fmt" "os" "path/filepath" "sync" "time" - "github.com/databricks/cli/bundle" - + "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/log" ) +// Metrics is a local interface for tracking cache telemetry. +type Metrics interface { + SetBoolValue(key string, value bool) +} + // FileCache implements the Cache interface using local disk storage. type FileCache[T any] struct { baseDir string expiryMinutes int - mu sync.RWMutex - computeOnce map[string]*sync.Once // Ensure only one goroutine computes per key - memCache map[string]T // In-memory cache for immediate access - cleanupMgr *CleanupManager // Background cleanup manager - metrics *bundle.Metrics // Telemetry metrics + mu sync.Mutex + metrics Metrics } // newFileCacheWithBaseDir creates a new file-based cache that stores data in the specified directory. func newFileCacheWithBaseDir[T any](baseDir string, expiryMinutes int) (*FileCache[T], error) { - if err := os.MkdirAll(baseDir, 0o755); err != nil { + if err := os.MkdirAll(baseDir, 0o700); err != nil { return nil, fmt.Errorf("failed to create cache directory: %w", err) } - cleanupMgr := NewCleanupManager(DefaultCleanupConfig()) - fc := &FileCache[T]{ baseDir: baseDir, expiryMinutes: expiryMinutes, - computeOnce: make(map[string]*sync.Once), - memCache: make(map[string]T), - cleanupMgr: cleanupMgr, } - // Start background cleanup (non-blocking) - cleanupMgr.Start(context.Background(), baseDir) + // Clean up expired files synchronously + fc.cleanupExpiredFiles() return fc, nil } +// cleanupExpiredFiles removes expired cache files from disk. +// This runs synchronously once when the cache is created. +func (fc *FileCache[T]) cleanupExpiredFiles() { + now := time.Now() + + _ = filepath.Walk(fc.baseDir, func(path string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() { + return nil + } + + // Only process .json cache files + if filepath.Ext(info.Name()) != ".json" { + return nil + } + + // Try to read the cache entry + data, err := os.ReadFile(path) + if err != nil { + return nil + } + + var entry cacheEntry + if err := json.Unmarshal(data, &entry); err != nil { + // Delete corrupted files + _ = os.Remove(path) + return nil + } + + // Delete if expired + if !entry.Expiry.IsZero() && now.After(entry.Expiry) { + _ = os.Remove(path) + } + + return nil + }) +} + func getCacheBaseDir() (string, error) { // Check if user has configured a custom cache directory - if customCacheDir := os.Getenv("DATABRICKS_CACHE_FOLDER"); customCacheDir != "" { + if customCacheDir := os.Getenv("DATABRICKS_CACHE_DIR"); customCacheDir != "" { return customCacheDir, nil } @@ -64,14 +94,17 @@ func getCacheBaseDir() (string, error) { return filepath.Join(userCacheDir, "databricks"), nil } -// NewFileCache creates a new file-based cache using UserCacheDir() + "databricks" + cached component name. -func NewFileCache[T any](component string, expiryMinutes int, metrics *bundle.Metrics) (*FileCache[T], error) { +// NewFileCache creates a new file-based cache using UserCacheDir() + "databricks" + version + cached component name. +// Including the CLI version in the path ensures cache isolation across different CLI versions. +func NewFileCache[T any](component string, expiryMinutes int, metrics Metrics) (*FileCache[T], error) { cacheBaseDir, err := getCacheBaseDir() if err != nil { return nil, err } - baseDir := filepath.Join(cacheBaseDir, component) + // Include CLI version in cache path to avoid issues across versions + version := build.GetInfo().Version + baseDir := filepath.Join(cacheBaseDir, version, component) fc, err := newFileCacheWithBaseDir[T](baseDir, expiryMinutes) if err != nil { return nil, err @@ -82,9 +115,8 @@ func NewFileCache[T any](component string, expiryMinutes int, metrics *bundle.Me // cacheEntry represents the structure of a cached item on disk. type cacheEntry struct { - Data json.RawMessage `json:"data"` - Expiry time.Time `json:"expiry"` - Timestamp time.Time `json:"timestamp,omitempty"` // For backward compatibility + Data json.RawMessage `json:"data"` + Expiry time.Time `json:"expiry"` } func (fc *FileCache[T]) addTelemetryMetric(key string) { @@ -94,13 +126,14 @@ func (fc *FileCache[T]) addTelemetryMetric(key string) { } // GetOrCompute retrieves cached content or computes it using the provided function. +// Cache operations fail open: if caching fails, the compute function is still called. func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compute func(ctx context.Context) (T, error)) (T, error) { - var zero T - // Convert fingerprint to deterministic hash - this is our cache key cacheKey, err := fingerprintToHash(fingerprint) if err != nil { - return zero, fmt.Errorf("failed to convert fingerprint to string: %w", err) + // Fail open: if we can't generate cache key, just compute directly + log.Debugf(ctx, "[Local Cache] failed to generate cache key, computing without cache: %v\n", err) + return compute(ctx) } log.Debugf(ctx, "[Local Cache] using cache key: %s\n", cacheKey) @@ -108,92 +141,37 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu cachePath := fc.getCachePath(cacheKey) - // Check in-memory cache first (fast path) - fc.mu.RLock() - if data, found := fc.memCache[cacheKey]; found { - fc.mu.RUnlock() - log.Debugf(ctx, "[Local Cache] cache hit: in-memory\n") - fc.addTelemetryMetric("local.cache.hit") - return data, nil - } - fc.mu.RUnlock() - // Try to read from disk cache if data, found := fc.readFromCache(cachePath); found { - // Store in memory cache for faster future access - fc.mu.Lock() - fc.memCache[cacheKey] = data - fc.mu.Unlock() - log.Debugf(ctx, "[Local Cache] cache hit: disk-read\n") + log.Debugf(ctx, "[Local Cache] cache hit\n") fc.addTelemetryMetric("local.cache.hit") return data, nil } - // Get or create sync.Once for this cache key - // Check cache again under write lock to avoid race condition + // Cache miss - acquire lock to compute fc.mu.Lock() - if data, found := fc.memCache[cacheKey]; found { - fc.mu.Unlock() - log.Debugf(ctx, "[Local Cache] cache hit: in-memory (race avoided)\n") + defer fc.mu.Unlock() + + // Check again after acquiring lock (another goroutine might have computed it) + if data, found := fc.readFromCache(cachePath); found { + log.Debugf(ctx, "[Local Cache] cache hit after lock\n") fc.addTelemetryMetric("local.cache.hit") return data, nil } - once, exists := fc.computeOnce[cacheKey] - if !exists { - once = &sync.Once{} - fc.computeOnce[cacheKey] = once - } - fc.mu.Unlock() - - // Use sync.Once to ensure only one goroutine computes the value - // Store error in a separate variable that all goroutines can access - var computeErr error - once.Do(func() { - // Check if context is already cancelled before computing - select { - case <-ctx.Done(): - log.Debugf(ctx, "[Local Cache] context cancelled before compute\n") - computeErr = ctx.Err() - return - default: - } - - // Compute the value - result, err := compute(ctx) - if err != nil { - log.Debugf(ctx, "[Local Cache] error while computing: %v\n", err) - fc.addTelemetryMetric("local.cache.error") - computeErr = err - return - } - - // Store in memory cache immediately - fc.mu.Lock() - fc.memCache[cacheKey] = result - fc.mu.Unlock() - // Write to disk cache synchronously to ensure it persists before process exits - log.Debugf(ctx, "[Local Cache] writing to cache\n") - fc.writeToCache(cachePath, result) - - log.Debugf(ctx, "[Local Cache] cache miss, computed and stored result\n") - fc.addTelemetryMetric("local.cache.miss") - }) - - // Check if computation failed - if computeErr != nil { - return zero, computeErr + // Compute the value + log.Debugf(ctx, "[Local Cache] cache miss, computing\n") + result, err := compute(ctx) + if err != nil { + log.Debugf(ctx, "[Local Cache] error while computing: %v\n", err) + fc.addTelemetryMetric("local.cache.error") + return result, err } - // All goroutines retrieve the result from memCache after sync.Once completes - fc.mu.RLock() - result, found := fc.memCache[cacheKey] - fc.mu.RUnlock() - - if !found { - // This should never happen unless there was an error - return zero, errors.New("cache inconsistency: value not found after computation") - } + // Write to disk cache (failures are silent - cache write errors don't affect the result) + fc.writeToCache(cachePath, result) + log.Debugf(ctx, "[Local Cache] computed and stored result\n") + fc.addTelemetryMetric("local.cache.miss") return result, nil } @@ -225,7 +203,7 @@ func (fc *FileCache[T]) readFromCache(cachePath string) (T, bool) { return result, true } -// writeToCache serializes and writes data to the cache file asynchronously. +// writeToCache serializes and writes data to the cache file. func (fc *FileCache[T]) writeToCache(cachePath string, data any) { // Serialize the data serializedData, err := json.Marshal(data) @@ -244,43 +222,15 @@ func (fc *FileCache[T]) writeToCache(cachePath string, data any) { } // Ensure directory exists - if err := os.MkdirAll(filepath.Dir(cachePath), 0o755); err != nil { + if err := os.MkdirAll(filepath.Dir(cachePath), 0o700); err != nil { return } - // Write to temporary file first, then rename for atomic operation - tempPath, err := generateTempPath(cachePath) - if err != nil { - return - } - - if err := os.WriteFile(tempPath, entryData, 0o644); err != nil { - return - } - - // Atomic rename - _ = os.Rename(tempPath, cachePath) -} - -// generateTempPath creates a temporary file path with a random component to prevent collisions. -func generateTempPath(cachePath string) (string, error) { - randomBytes := make([]byte, 8) - if _, err := rand.Read(randomBytes); err != nil { - return "", err - } - randomSuffix := hex.EncodeToString(randomBytes) - return cachePath + ".tmp." + randomSuffix, nil + // Write to cache file + _ = os.WriteFile(cachePath, entryData, 0o600) } // getCachePath returns the full path to the cache file for a given cache key. func (fc *FileCache[T]) getCachePath(cacheKey string) string { return filepath.Join(fc.baseDir, cacheKey+".json") } - -// StopCleanup stops the background cleanup process. -// This is non-blocking and will not wait for cleanup to complete. -func (fc *FileCache[T]) StopCleanup() { - if fc.cleanupMgr != nil { - fc.cleanupMgr.Stop() - } -} diff --git a/libs/cache/file_cache_expiry_test.go b/libs/cache/file_cache_expiry_test.go index deac7342e42..b137cd945a1 100644 --- a/libs/cache/file_cache_expiry_test.go +++ b/libs/cache/file_cache_expiry_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" ) -// TestFileCacheExpiryBehavior tests that the new expiry-based cache works as expected +// TestFileCacheExpiryBehavior tests that the cache writes files with correct expiry func TestFileCacheExpiryBehavior(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() @@ -20,7 +20,6 @@ func TestFileCacheExpiryBehavior(t *testing.T) { // Create cache with 1 minute expiry cache, err := newFileCacheWithBaseDir[string](tempDir, 1) require.NoError(t, err) - defer cache.StopCleanup() fingerprint := struct { Key string `json:"key"` @@ -35,9 +34,6 @@ func TestFileCacheExpiryBehavior(t *testing.T) { require.NoError(t, err) assert.Equal(t, "test-value", result) - // Allow time for async write to complete - time.Sleep(100 * time.Millisecond) - // Find the cache file and verify it has the correct expiry cacheFiles, err := filepath.Glob(filepath.Join(tempDir, "*.json")) require.NoError(t, err) @@ -56,38 +52,6 @@ func TestFileCacheExpiryBehavior(t *testing.T) { expectedExpiry := time.Now().Add(time.Minute) timeDiff := entry.Expiry.Sub(expectedExpiry).Abs() assert.Less(t, timeDiff, 10*time.Second, "Expiry should be approximately 1 minute from creation time") - - // Verify cleanup would identify an expired file - manager := NewCleanupManager(DefaultCleanupConfig()) - futureTime := time.Now().Add(2 * time.Minute) // 2 minutes from now, past expiry - shouldDelete, age := manager.shouldDeleteFile(cacheFiles[0], futureTime) - assert.True(t, shouldDelete, "File should be marked for deletion when past expiry") - assert.GreaterOrEqual(t, age, time.Duration(0), "Age should be positive when expired") -} - -// TestLegacyTimestampCompatibility tests that old cache files with timestamp still work -func TestLegacyTimestampCompatibility(t *testing.T) { - tempDir := t.TempDir() - - // Create a legacy cache file with timestamp - legacyEntry := cacheEntry{ - Data: json.RawMessage(`"legacy-value"`), - Timestamp: time.Now().Add(-time.Hour), // 1 hour ago - } - legacyData, err := json.Marshal(legacyEntry) - require.NoError(t, err) - - legacyFile := filepath.Join(tempDir, "legacy.json") - require.NoError(t, os.WriteFile(legacyFile, legacyData, 0o644)) - - // Test cleanup logic handles legacy files correctly - manager := NewCleanupManager(DefaultCleanupConfig()) - now := time.Now() - - // Should not delete a 1-hour-old file (default MaxAge is 7 days) - shouldDelete, age := manager.shouldDeleteFile(legacyFile, now) - assert.False(t, shouldDelete, "Legacy file should not be deleted if within MaxAge") - assert.GreaterOrEqual(t, age, time.Hour, "Age should be calculated from timestamp") } // TestReadFromCacheRespectsExpiry tests that readFromCache returns false for expired entries @@ -95,7 +59,6 @@ func TestReadFromCacheRespectsExpiry(t *testing.T) { tempDir := t.TempDir() cache, err := newFileCacheWithBaseDir[string](tempDir, 1) require.NoError(t, err) - defer cache.StopCleanup() // Create an expired cache file expiredEntry := cacheEntry{ diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index dc433ed8c51..4fa01a65990 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -2,6 +2,7 @@ package cache import ( "context" + "encoding/json" "os" "path/filepath" "runtime" @@ -17,11 +18,10 @@ func TestNewFileCache(t *testing.T) { tempDir := t.TempDir() cacheDir := filepath.Join(tempDir, "cache") - cache, err := newFileCacheWithBaseDir[string](cacheDir, 60) // 1 hour for tests + cache, err := newFileCacheWithBaseDir[string](cacheDir, 60) require.NoError(t, err) assert.NotNil(t, cache) assert.Equal(t, cacheDir, cache.baseDir) - assert.NotNil(t, cache.computeOnce) // Verify directory was created info, err := os.Stat(cacheDir) @@ -30,14 +30,14 @@ func TestNewFileCache(t *testing.T) { // Check permissions - Windows has different permission semantics if runtime.GOOS != "windows" { - assert.Equal(t, os.FileMode(0o755), info.Mode().Perm()) + assert.Equal(t, os.FileMode(0o700), info.Mode().Perm()) } else { // On Windows, verify directory is accessible by trying to create a test file testFile := filepath.Join(cacheDir, "test_access") - err := os.WriteFile(testFile, []byte("test"), 0o644) + err := os.WriteFile(testFile, []byte("test"), 0o600) assert.NoError(t, err) if err == nil { - _ = os.Remove(testFile) // Clean up (ignore removal error) + _ = os.Remove(testFile) } } } @@ -102,9 +102,6 @@ func TestFileCacheGetOrCompute(t *testing.T) { require.NoError(t, err) assert.Equal(t, expectedValue, result2) assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) - - // Allow time for async writes to complete before test cleanup - time.Sleep(50 * time.Millisecond) } func TestFileCacheGetOrComputeError(t *testing.T) { @@ -168,39 +165,66 @@ func TestFileCacheGetOrComputeConcurrency(t *testing.T) { assert.Equal(t, expectedValue, result) } - // With sync.Once, compute should only be called once even with concurrent requests + // With locking, compute should only be called once even with concurrent requests assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) - - // Allow time for async writes to complete before test cleanup - time.Sleep(50 * time.Millisecond) } -func TestFileCacheGetOrComputeContextCancellation(t *testing.T) { +func TestFileCacheCleanupExpiredFiles(t *testing.T) { tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](tempDir, 60) // 1 hour for tests - require.NoError(t, err) - ctx, cancel := context.WithCancel(context.Background()) - cancel() // Cancel immediately + // Create some cache files manually - one expired, one valid, one corrupted + now := time.Now() - fingerprint := struct { - Key string `json:"key"` - }{ - Key: "cancelled-key", + // Expired file + expiredEntry := cacheEntry{ + Data: json.RawMessage(`"expired-value"`), + Expiry: now.Add(-time.Hour), // Expired 1 hour ago } + expiredData, err := json.Marshal(expiredEntry) + require.NoError(t, err) + expiredFile := filepath.Join(tempDir, "expired.json") + require.NoError(t, os.WriteFile(expiredFile, expiredData, 0o644)) - result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { - return "should-not-be-reached", nil - }) + // Valid file + validEntry := cacheEntry{ + Data: json.RawMessage(`"valid-value"`), + Expiry: now.Add(time.Hour), // Expires in 1 hour + } + validData, err := json.Marshal(validEntry) + require.NoError(t, err) + validFile := filepath.Join(tempDir, "valid.json") + require.NoError(t, os.WriteFile(validFile, validData, 0o644)) - assert.Empty(t, result) - assert.Equal(t, context.Canceled, err) + // Corrupted file + corruptedFile := filepath.Join(tempDir, "corrupted.json") + require.NoError(t, os.WriteFile(corruptedFile, []byte("invalid json"), 0o644)) + + // Non-cache file (should be ignored) + nonCacheFile := filepath.Join(tempDir, "readme.txt") + require.NoError(t, os.WriteFile(nonCacheFile, []byte("readme"), 0o644)) + + // Create cache - this should trigger cleanup + _, err = newFileCacheWithBaseDir[string](tempDir, 60) + require.NoError(t, err) + + // Check results + _, err = os.Stat(expiredFile) + assert.True(t, os.IsNotExist(err), "Expired file should be deleted") + + _, err = os.Stat(validFile) + assert.False(t, os.IsNotExist(err), "Valid file should still exist") + + _, err = os.Stat(corruptedFile) + assert.True(t, os.IsNotExist(err), "Corrupted file should be deleted") + + _, err = os.Stat(nonCacheFile) + assert.False(t, os.IsNotExist(err), "Non-cache file should be ignored") } func TestFingerprintDeterministic(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](tempDir, 60) // 1 hour for tests + cache, err := newFileCacheWithBaseDir[string](tempDir, 60) require.NoError(t, err) // Create two identical structs with fields in different JSON order @@ -245,7 +269,4 @@ func TestFingerprintDeterministic(t *testing.T) { assert.Equal(t, expectedValue, result2) assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) // Should still be 1 - - // Allow time for async writes to complete before test cleanup - time.Sleep(50 * time.Millisecond) } From 367a6bfae11b3a980909823da952b63de9062762 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 17 Nov 2025 12:53:41 +0100 Subject: [PATCH 62/87] sanitise version --- libs/cache/file_cache.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 431f233d39c..3a2b5d4aa1a 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "sync" "time" @@ -94,6 +95,18 @@ func getCacheBaseDir() (string, error) { return filepath.Join(userCacheDir, "databricks"), nil } +// sanitizeVersion removes characters from version string that might be problematic in file paths. +// Particularly important for Windows which has restrictions on certain characters. +func sanitizeVersion(version string) string { + // Replace + with - (used in version metadata like "1.0.0+abc123") + version = strings.ReplaceAll(version, "+", "-") + // Remove any other potentially problematic characters + version = strings.ReplaceAll(version, ":", "-") + version = strings.ReplaceAll(version, "/", "-") + version = strings.ReplaceAll(version, "\\", "-") + return version +} + // NewFileCache creates a new file-based cache using UserCacheDir() + "databricks" + version + cached component name. // Including the CLI version in the path ensures cache isolation across different CLI versions. func NewFileCache[T any](component string, expiryMinutes int, metrics Metrics) (*FileCache[T], error) { @@ -103,7 +116,8 @@ func NewFileCache[T any](component string, expiryMinutes int, metrics Metrics) ( } // Include CLI version in cache path to avoid issues across versions - version := build.GetInfo().Version + // Sanitize version string for use in file paths + version := sanitizeVersion(build.GetInfo().Version) baseDir := filepath.Join(cacheBaseDir, version, component) fc, err := newFileCacheWithBaseDir[T](baseDir, expiryMinutes) if err != nil { From ae896822eed6a7c0dbf1a02396e6f37f0abd3c71 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 17 Nov 2025 14:21:49 +0100 Subject: [PATCH 63/87] use DATABRICKS_CACHE_DIR in tests --- acceptance/acceptance_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index bef08724411..b61e9f4db98 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -624,7 +624,7 @@ func runTest(t *testing.T, userCacheDir, err := os.UserCacheDir() require.NoError(t, err) uniqueCacheDir := filepath.Join(userCacheDir, strings.ReplaceAll(dir, string(os.PathSeparator), "--")) - cmd.Env = append(cmd.Env, "DATABRICKS_CACHE_FOLDER="+uniqueCacheDir) + cmd.Env = append(cmd.Env, "DATABRICKS_CACHE_DIR="+uniqueCacheDir) for _, key := range utils.SortedKeys(config.Env) { if hasKey(customEnv, key) { From 314852e69f81d9b8de8b9b637ffeb772e0c1fb83 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 25 Nov 2025 12:21:44 +0100 Subject: [PATCH 64/87] address comments --- libs/cache/cache.go | 21 ++----- libs/cache/file_cache.go | 87 ++++++++++++++-------------- libs/cache/file_cache_expiry_test.go | 48 +++++---------- libs/cache/file_cache_test.go | 36 ++++-------- 4 files changed, 75 insertions(+), 117 deletions(-) diff --git a/libs/cache/cache.go b/libs/cache/cache.go index 82299b83a0f..2edfc50f706 100644 --- a/libs/cache/cache.go +++ b/libs/cache/cache.go @@ -23,28 +23,15 @@ type Cache[T any] interface { } // fingerprintToHash converts any struct to a deterministic string representation for use as a cache key. -// For structs, json.Marshal uses struct field order, not JSON tag order. To ensure deterministic -// hashing regardless of struct field order, we convert to a map which json.Marshal sorts by key. func fingerprintToHash(fingerprint any) (string, error) { - // Marshal to JSON + // Marshal map - json.Marshal sorts map keys alphabetically data, err := json.Marshal(fingerprint) - if err != nil { - return "", fmt.Errorf("failed to marshal fingerprint: %w", err) - } - - // Unmarshal to map to ensure key ordering - var m map[string]any - if err := json.Unmarshal(data, &m); err != nil { - return "", fmt.Errorf("failed to unmarshal fingerprint: %w", err) - } - - // Marshal map (map keys are sorted by json.Marshal) - normalizedData, err := json.Marshal(m) if err != nil { return "", fmt.Errorf("failed to marshal normalized fingerprint: %w", err) } - // Hash for consistent, reasonably-sized key - hash := sha256.Sum256(normalizedData) + // Hash for consistent, reasonably-sized key. + // hash[:] converts the [32]byte array returned by Sum256 to a []byte slice. + hash := sha256.Sum256(data) return hex.EncodeToString(hash[:]), nil } diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 3a2b5d4aa1a..26d526bbae4 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -44,41 +44,44 @@ func newFileCacheWithBaseDir[T any](baseDir string, expiryMinutes int) (*FileCac return fc, nil } -// cleanupExpiredFiles removes expired cache files from disk. +// cleanupExpiredFiles removes expired cache files from disk based on file modification time. // This runs synchronously once when the cache is created. +// Files older than expiryMinutes are deleted. func (fc *FileCache[T]) cleanupExpiredFiles() { now := time.Now() + expiryDuration := time.Duration(fc.expiryMinutes) * time.Minute - _ = filepath.Walk(fc.baseDir, func(path string, info os.FileInfo, err error) error { - if err != nil || info.IsDir() { - return nil - } - - // Only process .json cache files - if filepath.Ext(info.Name()) != ".json" { + err := filepath.Walk(fc.baseDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + // Log walk errors but continue cleanup + log.Debugf(context.Background(), "[Local Cache] cleanup: failed to access path %s: %v", path, err) return nil } - // Try to read the cache entry - data, err := os.ReadFile(path) - if err != nil { + if info.IsDir() { return nil } - var entry cacheEntry - if err := json.Unmarshal(data, &entry); err != nil { - // Delete corrupted files - _ = os.Remove(path) + // Only process .json cache files + if filepath.Ext(info.Name()) != ".json" { return nil } - // Delete if expired - if !entry.Expiry.IsZero() && now.After(entry.Expiry) { - _ = os.Remove(path) + // Check if file is expired based on modification time + age := now.Sub(info.ModTime()) + if age > expiryDuration { + if err := os.Remove(path); err != nil { + log.Debugf(context.Background(), "[Local Cache] cleanup: failed to remove expired file %s: %v", path, err) + } else { + log.Debugf(context.Background(), "[Local Cache] cleanup: removed expired file %s (age: %v)", path, age) + } } return nil }) + if err != nil { + log.Warnf(context.Background(), "[Local Cache] cleanup: failed to walk cache directory: %v", err) + } } func getCacheBaseDir() (string, error) { @@ -127,11 +130,8 @@ func NewFileCache[T any](component string, expiryMinutes int, metrics Metrics) ( return fc, nil } -// cacheEntry represents the structure of a cached item on disk. -type cacheEntry struct { - Data json.RawMessage `json:"data"` - Expiry time.Time `json:"expiry"` -} +// Cache files are stored as JSON directly without metadata wrapper. +// Expiry is tracked using file modification time, not stored in the file itself. func (fc *FileCache[T]) addTelemetryMetric(key string) { if fc.metrics != nil { @@ -191,26 +191,33 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu } // readFromCache attempts to read and deserialize data from the cache file. +// Expiry is checked using file modification time for consistency with cleanup. func (fc *FileCache[T]) readFromCache(cachePath string) (T, bool) { var zero T - data, err := os.ReadFile(cachePath) + // Check file modification time for expiry + info, err := os.Stat(cachePath) if err != nil { + log.Debugf(context.Background(), "[Local Cache] failed to stat cache file: %v\n", err) return zero, false } - var entry cacheEntry - if err := json.Unmarshal(data, &entry); err != nil { + age := time.Since(info.ModTime()) + expiryDuration := time.Duration(fc.expiryMinutes) * time.Minute + if age > expiryDuration { return zero, false } - // Check if cache entry has expired - if time.Now().After(entry.Expiry) { + // Read and deserialize the data + data, err := os.ReadFile(cachePath) + if err != nil { + log.Debugf(context.Background(), "[Local Cache] failed to read cache file: %v\n", err) return zero, false } var result T - if err := json.Unmarshal(entry.Data, &result); err != nil { + if err := json.Unmarshal(data, &result); err != nil { + log.Debugf(context.Background(), "[Local Cache] failed to deserialize data: %v\n", err) return zero, false } @@ -218,30 +225,26 @@ func (fc *FileCache[T]) readFromCache(cachePath string) (T, bool) { } // writeToCache serializes and writes data to the cache file. +// Expiry is tracked by file modification time, not stored in the file. func (fc *FileCache[T]) writeToCache(cachePath string, data any) { - // Serialize the data + // Serialize the data directly serializedData, err := json.Marshal(data) if err != nil { - return // Silently fail on serialization errors - } - - entry := cacheEntry{ - Data: serializedData, - Expiry: time.Now().Add(time.Duration(fc.expiryMinutes) * time.Minute), - } - - entryData, err := json.Marshal(entry) - if err != nil { + log.Debugf(context.Background(), "[Local Cache] failed to serialize data: %v\n", err) return // Silently fail on serialization errors } // Ensure directory exists if err := os.MkdirAll(filepath.Dir(cachePath), 0o700); err != nil { + log.Debugf(context.Background(), "[Local Cache] failed to create directory: %v\n", err) return } - // Write to cache file - _ = os.WriteFile(cachePath, entryData, 0o600) + // Write to cache file - the mtime will be used to track expiry + err = os.WriteFile(cachePath, serializedData, 0o600) + if err != nil { + log.Debugf(context.Background(), "[Local Cache] failed to write to cache file: %v\n", err) + } } // getCachePath returns the full path to the cache file for a given cache key. diff --git a/libs/cache/file_cache_expiry_test.go b/libs/cache/file_cache_expiry_test.go index b137cd945a1..670e53395d1 100644 --- a/libs/cache/file_cache_expiry_test.go +++ b/libs/cache/file_cache_expiry_test.go @@ -2,7 +2,6 @@ package cache import ( "context" - "encoding/json" "os" "path/filepath" "testing" @@ -12,7 +11,7 @@ import ( "github.com/stretchr/testify/require" ) -// TestFileCacheExpiryBehavior tests that the cache writes files with correct expiry +// TestFileCacheExpiryBehavior tests that the cache writes files and respects expiry based on mtime func TestFileCacheExpiryBehavior(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() @@ -34,58 +33,43 @@ func TestFileCacheExpiryBehavior(t *testing.T) { require.NoError(t, err) assert.Equal(t, "test-value", result) - // Find the cache file and verify it has the correct expiry + // Find the cache file and verify it was created cacheFiles, err := filepath.Glob(filepath.Join(tempDir, "*.json")) require.NoError(t, err) require.Len(t, cacheFiles, 1) - // Read the cache file and check expiry + // Verify the file contains the expected data (stored directly, not wrapped) data, err := os.ReadFile(cacheFiles[0]) require.NoError(t, err) + assert.Equal(t, `"test-value"`, string(data)) - var entry cacheEntry - err = json.Unmarshal(data, &entry) + // Verify mtime is recent (within last 10 seconds) + info, err := os.Stat(cacheFiles[0]) require.NoError(t, err) - - // Verify expiry is set and is approximately 1 minute from now - assert.False(t, entry.Expiry.IsZero(), "Expiry should be set") - expectedExpiry := time.Now().Add(time.Minute) - timeDiff := entry.Expiry.Sub(expectedExpiry).Abs() - assert.Less(t, timeDiff, 10*time.Second, "Expiry should be approximately 1 minute from creation time") + age := time.Since(info.ModTime()) + assert.Less(t, age, 10*time.Second, "File should have been created recently") } -// TestReadFromCacheRespectsExpiry tests that readFromCache returns false for expired entries +// TestReadFromCacheRespectsExpiry tests that readFromCache returns false for expired entries based on mtime func TestReadFromCacheRespectsExpiry(t *testing.T) { tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](tempDir, 1) - require.NoError(t, err) - - // Create an expired cache file - expiredEntry := cacheEntry{ - Data: json.RawMessage(`"expired-value"`), - Expiry: time.Now().Add(-time.Hour), // Expired 1 hour ago - } - expiredData, err := json.Marshal(expiredEntry) + cache, err := newFileCacheWithBaseDir[string](tempDir, 1) // 1 minute expiry require.NoError(t, err) + // Create an expired cache file by setting its mtime to 2 hours ago expiredFile := filepath.Join(tempDir, "expired.json") - require.NoError(t, os.WriteFile(expiredFile, expiredData, 0o644)) + require.NoError(t, os.WriteFile(expiredFile, []byte(`"expired-value"`), 0o644)) + oldTime := time.Now().Add(-2 * time.Hour) + require.NoError(t, os.Chtimes(expiredFile, oldTime, oldTime)) // Try to read from expired cache - should return false result, found := cache.readFromCache(expiredFile) assert.False(t, found, "Should not find expired cache entry") assert.Equal(t, "", result, "Result should be zero value for expired entry") - // Create a valid (non-expired) cache file - validEntry := cacheEntry{ - Data: json.RawMessage(`"valid-value"`), - Expiry: time.Now().Add(time.Hour), // Expires in 1 hour - } - validData, err := json.Marshal(validEntry) - require.NoError(t, err) - + // Create a valid (non-expired) cache file with recent mtime validFile := filepath.Join(tempDir, "valid.json") - require.NoError(t, os.WriteFile(validFile, validData, 0o644)) + require.NoError(t, os.WriteFile(validFile, []byte(`"valid-value"`), 0o644)) // Try to read from valid cache - should return true result, found = cache.readFromCache(validFile) diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index 4fa01a65990..058f09f9de2 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -2,7 +2,6 @@ package cache import ( "context" - "encoding/json" "os" "path/filepath" "runtime" @@ -171,40 +170,28 @@ func TestFileCacheGetOrComputeConcurrency(t *testing.T) { func TestFileCacheCleanupExpiredFiles(t *testing.T) { tempDir := t.TempDir() + expiryMinutes := 60 - // Create some cache files manually - one expired, one valid, one corrupted + // Create some cache files manually - one expired, one valid now := time.Now() - // Expired file - expiredEntry := cacheEntry{ - Data: json.RawMessage(`"expired-value"`), - Expiry: now.Add(-time.Hour), // Expired 1 hour ago - } - expiredData, err := json.Marshal(expiredEntry) - require.NoError(t, err) + // Expired file - create it and set mtime to make it appear old expiredFile := filepath.Join(tempDir, "expired.json") - require.NoError(t, os.WriteFile(expiredFile, expiredData, 0o644)) + require.NoError(t, os.WriteFile(expiredFile, []byte(`"expired-value"`), 0o644)) + // Set mtime to 2 hours ago (older than expiry) + oldTime := now.Add(-2 * time.Hour) + require.NoError(t, os.Chtimes(expiredFile, oldTime, oldTime)) - // Valid file - validEntry := cacheEntry{ - Data: json.RawMessage(`"valid-value"`), - Expiry: now.Add(time.Hour), // Expires in 1 hour - } - validData, err := json.Marshal(validEntry) - require.NoError(t, err) + // Valid file - recently created validFile := filepath.Join(tempDir, "valid.json") - require.NoError(t, os.WriteFile(validFile, validData, 0o644)) - - // Corrupted file - corruptedFile := filepath.Join(tempDir, "corrupted.json") - require.NoError(t, os.WriteFile(corruptedFile, []byte("invalid json"), 0o644)) + require.NoError(t, os.WriteFile(validFile, []byte(`"valid-value"`), 0o644)) // Non-cache file (should be ignored) nonCacheFile := filepath.Join(tempDir, "readme.txt") require.NoError(t, os.WriteFile(nonCacheFile, []byte("readme"), 0o644)) // Create cache - this should trigger cleanup - _, err = newFileCacheWithBaseDir[string](tempDir, 60) + _, err := newFileCacheWithBaseDir[string](tempDir, expiryMinutes) require.NoError(t, err) // Check results @@ -214,9 +201,6 @@ func TestFileCacheCleanupExpiredFiles(t *testing.T) { _, err = os.Stat(validFile) assert.False(t, os.IsNotExist(err), "Valid file should still exist") - _, err = os.Stat(corruptedFile) - assert.True(t, os.IsNotExist(err), "Corrupted file should be deleted") - _, err = os.Stat(nonCacheFile) assert.False(t, os.IsNotExist(err), "Non-cache file should be ignored") } From 5c2b570cc592d0cdf237fe6add51b8384ba4f0e1 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 25 Nov 2025 12:30:39 +0100 Subject: [PATCH 65/87] fix tests --- acceptance/cache/clear/output.txt | 4 +++ acceptance/cache/simple/output.txt | 2 ++ libs/cache/file_cache_test.go | 50 ------------------------------ 3 files changed, 6 insertions(+), 50 deletions(-) diff --git a/acceptance/cache/clear/output.txt b/acceptance/cache/clear/output.txt index 4ee4c6e8372..1a785f84750 100644 --- a/acceptance/cache/clear/output.txt +++ b/acceptance/cache/clear/output.txt @@ -3,6 +3,8 @@ [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] +[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-314852e69f81/auth/[SHA256_HASH].json: no such file or directory +[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-314852e69f81/auth/[SHA256_HASH].json: no such file or directory [DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing [DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result @@ -19,5 +21,7 @@ Cache cleared successfully from [TEST_TMP_DIR]/.cache [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] +[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-314852e69f81/auth/[SHA256_HASH].json: no such file or directory +[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-314852e69f81/auth/[SHA256_HASH].json: no such file or directory [DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing [DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result diff --git a/acceptance/cache/simple/output.txt b/acceptance/cache/simple/output.txt index d8810bbf73b..828b87d2978 100644 --- a/acceptance/cache/simple/output.txt +++ b/acceptance/cache/simple/output.txt @@ -3,6 +3,8 @@ [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] +[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-314852e69f81/auth/[SHA256_HASH].json: no such file or directory +[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-314852e69f81/auth/[SHA256_HASH].json: no such file or directory [DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing [DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index 058f09f9de2..6874e423c3a 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -204,53 +204,3 @@ func TestFileCacheCleanupExpiredFiles(t *testing.T) { _, err = os.Stat(nonCacheFile) assert.False(t, os.IsNotExist(err), "Non-cache file should be ignored") } - -func TestFingerprintDeterministic(t *testing.T) { - ctx := context.Background() - tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](tempDir, 60) - require.NoError(t, err) - - // Create two identical structs with fields in different JSON order - fingerprint1 := struct { - A string `json:"a"` - B int `json:"b"` - C bool `json:"c"` - }{ - A: "value1", - B: 42, - C: true, - } - - fingerprint2 := struct { - C bool `json:"c"` - A string `json:"a"` - B int `json:"b"` - }{ - C: true, - A: "value1", - B: 42, - } - - expectedValue := "deterministic-value" - var computeCalls int32 - - // First call with fingerprint1 - result1, err := cache.GetOrCompute(ctx, fingerprint1, func(ctx context.Context) (string, error) { - atomic.AddInt32(&computeCalls, 1) - return expectedValue, nil - }) - require.NoError(t, err) - assert.Equal(t, expectedValue, result1) - assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) - - // Second call with fingerprint2 (should hit cache due to deterministic hashing, not compute again) - result2, err := cache.GetOrCompute(ctx, fingerprint2, func(ctx context.Context) (string, error) { - atomic.AddInt32(&computeCalls, 1) - return "should-not-be-called", nil - }) - require.NoError(t, err) - - assert.Equal(t, expectedValue, result2) - assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) // Should still be 1 -} From 59b2d2a4b69dfecbc42ec776364ebce700c1e828 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 25 Nov 2025 12:37:43 +0100 Subject: [PATCH 66/87] repl --- acceptance/cache/clear/output.txt | 8 ++++---- acceptance/cache/clear/test.toml | 4 ++++ acceptance/cache/simple/output.txt | 4 ++-- acceptance/cache/simple/test.toml | 4 ++++ 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/acceptance/cache/clear/output.txt b/acceptance/cache/clear/output.txt index 1a785f84750..09720fd76b4 100644 --- a/acceptance/cache/clear/output.txt +++ b/acceptance/cache/clear/output.txt @@ -3,8 +3,8 @@ [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-314852e69f81/auth/[SHA256_HASH].json: no such file or directory -[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-314852e69f81/auth/[SHA256_HASH].json: no such file or directory +[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-[CACHE_KEY]/auth/[SHA256_HASH].json: no such file or directory +[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-[CACHE_KEY]/auth/[SHA256_HASH].json: no such file or directory [DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing [DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result @@ -21,7 +21,7 @@ Cache cleared successfully from [TEST_TMP_DIR]/.cache [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-314852e69f81/auth/[SHA256_HASH].json: no such file or directory -[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-314852e69f81/auth/[SHA256_HASH].json: no such file or directory +[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-[CACHE_KEY]/auth/[SHA256_HASH].json: no such file or directory +[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-[CACHE_KEY]/auth/[SHA256_HASH].json: no such file or directory [DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing [DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result diff --git a/acceptance/cache/clear/test.toml b/acceptance/cache/clear/test.toml index 2853ed8cf72..6fa45a3d6a9 100644 --- a/acceptance/cache/clear/test.toml +++ b/acceptance/cache/clear/test.toml @@ -8,3 +8,7 @@ New = "[DEBUG_TIMESTAMP]" [[Repls]] Old = '[a-f0-9]{64}' New = "[SHA256_HASH]" + +[[Repls]] +Old = '[a-f0-9]{12}' +New = "[CACHE_KEY]" diff --git a/acceptance/cache/simple/output.txt b/acceptance/cache/simple/output.txt index 828b87d2978..512f76d1cd7 100644 --- a/acceptance/cache/simple/output.txt +++ b/acceptance/cache/simple/output.txt @@ -3,8 +3,8 @@ [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-314852e69f81/auth/[SHA256_HASH].json: no such file or directory -[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-314852e69f81/auth/[SHA256_HASH].json: no such file or directory +[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-[CACHE_KEY]/auth/[SHA256_HASH].json: no such file or directory +[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-[CACHE_KEY]/auth/[SHA256_HASH].json: no such file or directory [DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing [DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result diff --git a/acceptance/cache/simple/test.toml b/acceptance/cache/simple/test.toml index f931adddecc..c12fb9e3676 100644 --- a/acceptance/cache/simple/test.toml +++ b/acceptance/cache/simple/test.toml @@ -10,3 +10,7 @@ New = "[DEBUG_TIMESTAMP]" [[Repls]] Old = '[a-f0-9]{64}' New = "[SHA256_HASH]" + +[[Repls]] +Old = '[a-f0-9]{12}' +New = "[CACHE_KEY]" From 828e41639ca81ca6ab64ae19bdc5c38ead1e67f3 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 25 Nov 2025 12:42:23 +0100 Subject: [PATCH 67/87] ctx --- .../config/mutator/populate_current_user.go | 2 +- libs/cache/file_cache.go | 40 +++++++++---------- libs/cache/file_cache_clear.go | 7 ---- libs/cache/file_cache_expiry_test.go | 9 +++-- libs/cache/file_cache_test.go | 18 +++++---- 5 files changed, 37 insertions(+), 39 deletions(-) diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index 067009ba342..e75998f67db 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -39,7 +39,7 @@ func (m *populateCurrentUser) initializeCache(ctx context.Context, b *bundle.Bun } var err error - m.cache, err = cache.NewFileCache[*iam.User]("auth", 30, &b.Metrics) + m.cache, err = cache.NewFileCache[*iam.User](ctx, "auth", 30, &b.Metrics) if err != nil { log.Debugf(ctx, "[Local Cache] Failed to initialize cache dir: %v\n", err) } diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 26d526bbae4..6b08a8af2eb 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -28,7 +28,7 @@ type FileCache[T any] struct { } // newFileCacheWithBaseDir creates a new file-based cache that stores data in the specified directory. -func newFileCacheWithBaseDir[T any](baseDir string, expiryMinutes int) (*FileCache[T], error) { +func newFileCacheWithBaseDir[T any](ctx context.Context, baseDir string, expiryMinutes int) (*FileCache[T], error) { if err := os.MkdirAll(baseDir, 0o700); err != nil { return nil, fmt.Errorf("failed to create cache directory: %w", err) } @@ -39,7 +39,7 @@ func newFileCacheWithBaseDir[T any](baseDir string, expiryMinutes int) (*FileCac } // Clean up expired files synchronously - fc.cleanupExpiredFiles() + fc.cleanupExpiredFiles(ctx) return fc, nil } @@ -47,14 +47,14 @@ func newFileCacheWithBaseDir[T any](baseDir string, expiryMinutes int) (*FileCac // cleanupExpiredFiles removes expired cache files from disk based on file modification time. // This runs synchronously once when the cache is created. // Files older than expiryMinutes are deleted. -func (fc *FileCache[T]) cleanupExpiredFiles() { +func (fc *FileCache[T]) cleanupExpiredFiles(ctx context.Context) { now := time.Now() expiryDuration := time.Duration(fc.expiryMinutes) * time.Minute err := filepath.Walk(fc.baseDir, func(path string, info os.FileInfo, err error) error { if err != nil { // Log walk errors but continue cleanup - log.Debugf(context.Background(), "[Local Cache] cleanup: failed to access path %s: %v", path, err) + log.Debugf(ctx, "[Local Cache] cleanup: failed to access path %s: %v", path, err) return nil } @@ -71,16 +71,16 @@ func (fc *FileCache[T]) cleanupExpiredFiles() { age := now.Sub(info.ModTime()) if age > expiryDuration { if err := os.Remove(path); err != nil { - log.Debugf(context.Background(), "[Local Cache] cleanup: failed to remove expired file %s: %v", path, err) + log.Debugf(ctx, "[Local Cache] cleanup: failed to remove expired file %s: %v", path, err) } else { - log.Debugf(context.Background(), "[Local Cache] cleanup: removed expired file %s (age: %v)", path, age) + log.Debugf(ctx, "[Local Cache] cleanup: removed expired file %s (age: %v)", path, age) } } return nil }) if err != nil { - log.Warnf(context.Background(), "[Local Cache] cleanup: failed to walk cache directory: %v", err) + log.Warnf(ctx, "[Local Cache] cleanup: failed to walk cache directory: %v", err) } } @@ -112,7 +112,7 @@ func sanitizeVersion(version string) string { // NewFileCache creates a new file-based cache using UserCacheDir() + "databricks" + version + cached component name. // Including the CLI version in the path ensures cache isolation across different CLI versions. -func NewFileCache[T any](component string, expiryMinutes int, metrics Metrics) (*FileCache[T], error) { +func NewFileCache[T any](ctx context.Context, component string, expiryMinutes int, metrics Metrics) (*FileCache[T], error) { cacheBaseDir, err := getCacheBaseDir() if err != nil { return nil, err @@ -122,7 +122,7 @@ func NewFileCache[T any](component string, expiryMinutes int, metrics Metrics) ( // Sanitize version string for use in file paths version := sanitizeVersion(build.GetInfo().Version) baseDir := filepath.Join(cacheBaseDir, version, component) - fc, err := newFileCacheWithBaseDir[T](baseDir, expiryMinutes) + fc, err := newFileCacheWithBaseDir[T](ctx, baseDir, expiryMinutes) if err != nil { return nil, err } @@ -156,7 +156,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu cachePath := fc.getCachePath(cacheKey) // Try to read from disk cache - if data, found := fc.readFromCache(cachePath); found { + if data, found := fc.readFromCache(ctx, cachePath); found { log.Debugf(ctx, "[Local Cache] cache hit\n") fc.addTelemetryMetric("local.cache.hit") return data, nil @@ -167,7 +167,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu defer fc.mu.Unlock() // Check again after acquiring lock (another goroutine might have computed it) - if data, found := fc.readFromCache(cachePath); found { + if data, found := fc.readFromCache(ctx, cachePath); found { log.Debugf(ctx, "[Local Cache] cache hit after lock\n") fc.addTelemetryMetric("local.cache.hit") return data, nil @@ -183,7 +183,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu } // Write to disk cache (failures are silent - cache write errors don't affect the result) - fc.writeToCache(cachePath, result) + fc.writeToCache(ctx, cachePath, result) log.Debugf(ctx, "[Local Cache] computed and stored result\n") fc.addTelemetryMetric("local.cache.miss") @@ -192,13 +192,13 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu // readFromCache attempts to read and deserialize data from the cache file. // Expiry is checked using file modification time for consistency with cleanup. -func (fc *FileCache[T]) readFromCache(cachePath string) (T, bool) { +func (fc *FileCache[T]) readFromCache(ctx context.Context, cachePath string) (T, bool) { var zero T // Check file modification time for expiry info, err := os.Stat(cachePath) if err != nil { - log.Debugf(context.Background(), "[Local Cache] failed to stat cache file: %v\n", err) + log.Debugf(ctx, "[Local Cache] failed to stat cache file: %v\n", err) return zero, false } @@ -211,13 +211,13 @@ func (fc *FileCache[T]) readFromCache(cachePath string) (T, bool) { // Read and deserialize the data data, err := os.ReadFile(cachePath) if err != nil { - log.Debugf(context.Background(), "[Local Cache] failed to read cache file: %v\n", err) + log.Debugf(ctx, "[Local Cache] failed to read cache file: %v\n", err) return zero, false } var result T if err := json.Unmarshal(data, &result); err != nil { - log.Debugf(context.Background(), "[Local Cache] failed to deserialize data: %v\n", err) + log.Debugf(ctx, "[Local Cache] failed to deserialize data: %v\n", err) return zero, false } @@ -226,24 +226,24 @@ func (fc *FileCache[T]) readFromCache(cachePath string) (T, bool) { // writeToCache serializes and writes data to the cache file. // Expiry is tracked by file modification time, not stored in the file. -func (fc *FileCache[T]) writeToCache(cachePath string, data any) { +func (fc *FileCache[T]) writeToCache(ctx context.Context, cachePath string, data any) { // Serialize the data directly serializedData, err := json.Marshal(data) if err != nil { - log.Debugf(context.Background(), "[Local Cache] failed to serialize data: %v\n", err) + log.Debugf(ctx, "[Local Cache] failed to serialize data: %v\n", err) return // Silently fail on serialization errors } // Ensure directory exists if err := os.MkdirAll(filepath.Dir(cachePath), 0o700); err != nil { - log.Debugf(context.Background(), "[Local Cache] failed to create directory: %v\n", err) + log.Debugf(ctx, "[Local Cache] failed to create directory: %v\n", err) return } // Write to cache file - the mtime will be used to track expiry err = os.WriteFile(cachePath, serializedData, 0o600) if err != nil { - log.Debugf(context.Background(), "[Local Cache] failed to write to cache file: %v\n", err) + log.Debugf(ctx, "[Local Cache] failed to write to cache file: %v\n", err) } } diff --git a/libs/cache/file_cache_clear.go b/libs/cache/file_cache_clear.go index e3d5bc1a4bf..c22dca00a4c 100644 --- a/libs/cache/file_cache_clear.go +++ b/libs/cache/file_cache_clear.go @@ -2,7 +2,6 @@ package cache import ( "context" - "fmt" "os" "github.com/databricks/cli/libs/cmdio" @@ -14,12 +13,6 @@ func ClearFileCache(ctx context.Context) error { return err } - // Check if the cache directory exists - if _, err := os.Stat(databricksCacheDir); os.IsNotExist(err) { - cmdio.LogString(ctx, fmt.Sprintf("No cache directory found at %s, nothing to clear", databricksCacheDir)) - return nil - } - // Remove the entire databricks cache directory err = os.RemoveAll(databricksCacheDir) if err != nil { diff --git a/libs/cache/file_cache_expiry_test.go b/libs/cache/file_cache_expiry_test.go index 670e53395d1..847cdd0e79a 100644 --- a/libs/cache/file_cache_expiry_test.go +++ b/libs/cache/file_cache_expiry_test.go @@ -17,7 +17,7 @@ func TestFileCacheExpiryBehavior(t *testing.T) { tempDir := t.TempDir() // Create cache with 1 minute expiry - cache, err := newFileCacheWithBaseDir[string](tempDir, 1) + cache, err := newFileCacheWithBaseDir[string](ctx, tempDir, 1) require.NoError(t, err) fingerprint := struct { @@ -52,8 +52,9 @@ func TestFileCacheExpiryBehavior(t *testing.T) { // TestReadFromCacheRespectsExpiry tests that readFromCache returns false for expired entries based on mtime func TestReadFromCacheRespectsExpiry(t *testing.T) { + ctx := context.Background() tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](tempDir, 1) // 1 minute expiry + cache, err := newFileCacheWithBaseDir[string](ctx, tempDir, 1) // 1 minute expiry require.NoError(t, err) // Create an expired cache file by setting its mtime to 2 hours ago @@ -63,7 +64,7 @@ func TestReadFromCacheRespectsExpiry(t *testing.T) { require.NoError(t, os.Chtimes(expiredFile, oldTime, oldTime)) // Try to read from expired cache - should return false - result, found := cache.readFromCache(expiredFile) + result, found := cache.readFromCache(ctx, expiredFile) assert.False(t, found, "Should not find expired cache entry") assert.Equal(t, "", result, "Result should be zero value for expired entry") @@ -72,7 +73,7 @@ func TestReadFromCacheRespectsExpiry(t *testing.T) { require.NoError(t, os.WriteFile(validFile, []byte(`"valid-value"`), 0o644)) // Try to read from valid cache - should return true - result, found = cache.readFromCache(validFile) + result, found = cache.readFromCache(ctx, validFile) assert.True(t, found, "Should find valid cache entry") assert.Equal(t, "valid-value", result, "Should return correct value for valid entry") } diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index 6874e423c3a..6dc7fba6802 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -14,10 +14,11 @@ import ( ) func TestNewFileCache(t *testing.T) { + ctx := context.Background() tempDir := t.TempDir() cacheDir := filepath.Join(tempDir, "cache") - cache, err := newFileCacheWithBaseDir[string](cacheDir, 60) + cache, err := newFileCacheWithBaseDir[string](ctx, cacheDir, 60) require.NoError(t, err) assert.NotNil(t, cache) assert.Equal(t, cacheDir, cache.baseDir) @@ -42,6 +43,7 @@ func TestNewFileCache(t *testing.T) { } func TestNewFileCacheWithExistingDirectory(t *testing.T) { + ctx := context.Background() tempDir := t.TempDir() cacheDir := filepath.Join(tempDir, "existing") @@ -49,17 +51,18 @@ func TestNewFileCacheWithExistingDirectory(t *testing.T) { err := os.MkdirAll(cacheDir, 0o700) require.NoError(t, err) - cache, err := newFileCacheWithBaseDir[string](cacheDir, 60) // 1 hour for tests + cache, err := newFileCacheWithBaseDir[string](ctx, cacheDir, 60) // 1 hour for tests require.NoError(t, err) assert.NotNil(t, cache) assert.Equal(t, cacheDir, cache.baseDir) } func TestNewFileCacheInvalidPath(t *testing.T) { + ctx := context.Background() // Try to create cache in a location that should fail invalidPath := "/root/invalid/path/that/should/not/exist" - cache, err := newFileCacheWithBaseDir[string](invalidPath, 60) // 1 hour for tests + cache, err := newFileCacheWithBaseDir[string](ctx, invalidPath, 60) // 1 hour for tests if err != nil { assert.Nil(t, cache) assert.Contains(t, err.Error(), "failed to create cache directory") @@ -69,7 +72,7 @@ func TestNewFileCacheInvalidPath(t *testing.T) { func TestFileCacheGetOrCompute(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](tempDir, 60) // 1 hour for tests + cache, err := newFileCacheWithBaseDir[string](ctx, tempDir, 60) // 1 hour for tests require.NoError(t, err) fingerprint := struct { @@ -106,7 +109,7 @@ func TestFileCacheGetOrCompute(t *testing.T) { func TestFileCacheGetOrComputeError(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](tempDir, 60) // 1 hour for tests + cache, err := newFileCacheWithBaseDir[string](ctx, tempDir, 60) // 1 hour for tests require.NoError(t, err) fingerprint := struct { @@ -128,7 +131,7 @@ func TestFileCacheGetOrComputeError(t *testing.T) { func TestFileCacheGetOrComputeConcurrency(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](tempDir, 60) // 1 hour for tests + cache, err := newFileCacheWithBaseDir[string](ctx, tempDir, 60) // 1 hour for tests require.NoError(t, err) fingerprint := struct { @@ -169,6 +172,7 @@ func TestFileCacheGetOrComputeConcurrency(t *testing.T) { } func TestFileCacheCleanupExpiredFiles(t *testing.T) { + ctx := context.Background() tempDir := t.TempDir() expiryMinutes := 60 @@ -191,7 +195,7 @@ func TestFileCacheCleanupExpiredFiles(t *testing.T) { require.NoError(t, os.WriteFile(nonCacheFile, []byte("readme"), 0o644)) // Create cache - this should trigger cleanup - _, err := newFileCacheWithBaseDir[string](tempDir, expiryMinutes) + _, err := newFileCacheWithBaseDir[string](ctx, tempDir, expiryMinutes) require.NoError(t, err) // Check results From 60770f190f9808f5504744271f7b632bfe7c17ac Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 25 Nov 2025 13:03:55 +0100 Subject: [PATCH 68/87] fix test for windows --- acceptance/cache/clear/output.txt | 8 ++++---- acceptance/cache/clear/test.toml | 12 ------------ acceptance/cache/simple/output.txt | 4 ++-- acceptance/cache/simple/test.toml | 12 ------------ acceptance/cache/test.toml | 15 +++++++++++++++ 5 files changed, 21 insertions(+), 30 deletions(-) create mode 100644 acceptance/cache/test.toml diff --git a/acceptance/cache/clear/output.txt b/acceptance/cache/clear/output.txt index 09720fd76b4..167c926d82d 100644 --- a/acceptance/cache/clear/output.txt +++ b/acceptance/cache/clear/output.txt @@ -3,8 +3,8 @@ [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-[CACHE_KEY]/auth/[SHA256_HASH].json: no such file or directory -[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-[CACHE_KEY]/auth/[SHA256_HASH].json: no such file or directory +[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted) +[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted) [DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing [DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result @@ -21,7 +21,7 @@ Cache cleared successfully from [TEST_TMP_DIR]/.cache [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-[CACHE_KEY]/auth/[SHA256_HASH].json: no such file or directory -[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-[CACHE_KEY]/auth/[SHA256_HASH].json: no such file or directory +[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted) +[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted) [DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing [DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result diff --git a/acceptance/cache/clear/test.toml b/acceptance/cache/clear/test.toml index 6fa45a3d6a9..df4bd1ceb8e 100644 --- a/acceptance/cache/clear/test.toml +++ b/acceptance/cache/clear/test.toml @@ -1,14 +1,2 @@ Cloud=false Local=true - -[[Repls]] -Old = '\d\d:\d\d:\d\d' -New = "[DEBUG_TIMESTAMP]" - -[[Repls]] -Old = '[a-f0-9]{64}' -New = "[SHA256_HASH]" - -[[Repls]] -Old = '[a-f0-9]{12}' -New = "[CACHE_KEY]" diff --git a/acceptance/cache/simple/output.txt b/acceptance/cache/simple/output.txt index 512f76d1cd7..1c5d30b76c3 100644 --- a/acceptance/cache/simple/output.txt +++ b/acceptance/cache/simple/output.txt @@ -3,8 +3,8 @@ [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] -[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-[CACHE_KEY]/auth/[SHA256_HASH].json: no such file or directory -[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: stat [TEST_TMP_DIR]/.cache/[DEV_VERSION]-[CACHE_KEY]/auth/[SHA256_HASH].json: no such file or directory +[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted) +[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted) [DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing [DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result diff --git a/acceptance/cache/simple/test.toml b/acceptance/cache/simple/test.toml index c12fb9e3676..97c8ceb226b 100644 --- a/acceptance/cache/simple/test.toml +++ b/acceptance/cache/simple/test.toml @@ -2,15 +2,3 @@ Cloud=false Local=true RecordRequests = true - -[[Repls]] -Old = '\d\d:\d\d:\d\d' -New = "[DEBUG_TIMESTAMP]" - -[[Repls]] -Old = '[a-f0-9]{64}' -New = "[SHA256_HASH]" - -[[Repls]] -Old = '[a-f0-9]{12}' -New = "[CACHE_KEY]" diff --git a/acceptance/cache/test.toml b/acceptance/cache/test.toml new file mode 100644 index 00000000000..ef8415db362 --- /dev/null +++ b/acceptance/cache/test.toml @@ -0,0 +1,15 @@ +[[Repls]] +Old = '\d\d:\d\d:\d\d' +New = "[DEBUG_TIMESTAMP]" + +[[Repls]] +Old = '[a-f0-9]{64}' +New = "[SHA256_HASH]" + +[[Repls]] +Old = '[a-f0-9]{12}' +New = "[CACHE_KEY]" + +[[Repls]] +Old = 'failed to stat cache file: .*' +New = "failed to stat cache file: (redacted)" From bcaf666974b89a06f0e37920d7b924852f5b7335 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 27 Nov 2025 12:59:41 +0100 Subject: [PATCH 69/87] only measure dont cache --- acceptance/cache/clear/out.test.toml | 1 + acceptance/cache/clear/output.txt | 5 +- acceptance/cache/clear/test.toml | 3 + acceptance/cache/simple/out.test.toml | 1 + acceptance/cache/simple/output.txt | 3 +- acceptance/cache/simple/test.toml | 3 + bundle/bundle.go | 17 +++++ .../config/mutator/populate_current_user.go | 12 +-- bundle/phases/telemetry.go | 1 + libs/cache/file_cache.go | 73 ++++++++++++++----- libs/cache/file_cache_test.go | 7 +- libs/telemetry/protos/bundle_deploy.go | 3 + 12 files changed, 98 insertions(+), 31 deletions(-) diff --git a/acceptance/cache/clear/out.test.toml b/acceptance/cache/clear/out.test.toml index d560f1de043..7af6fdc0a51 100644 --- a/acceptance/cache/clear/out.test.toml +++ b/acceptance/cache/clear/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + DATABRICKS_CACHE_ENABLED = ["true"] diff --git a/acceptance/cache/clear/output.txt b/acceptance/cache/clear/output.txt index 167c926d82d..8999e205531 100644 --- a/acceptance/cache/clear/output.txt +++ b/acceptance/cache/clear/output.txt @@ -1,14 +1,15 @@ === First call in a session is expected to be a cache miss: +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache is enabled for actual use [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted) -[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted) [DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing [DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result === Second call in a session is expected to be a cache hit +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache is enabled for actual use [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] @@ -18,10 +19,10 @@ Cache cleared successfully from [TEST_TMP_DIR]/.cache === First call after a clear is expected to be a cache miss: +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache is enabled for actual use [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted) -[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted) [DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing [DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result diff --git a/acceptance/cache/clear/test.toml b/acceptance/cache/clear/test.toml index df4bd1ceb8e..746f844de76 100644 --- a/acceptance/cache/clear/test.toml +++ b/acceptance/cache/clear/test.toml @@ -1,2 +1,5 @@ Cloud=false Local=true + +[EnvMatrix] + DATABRICKS_CACHE_ENABLED = ["true"] diff --git a/acceptance/cache/simple/out.test.toml b/acceptance/cache/simple/out.test.toml index d560f1de043..7af6fdc0a51 100644 --- a/acceptance/cache/simple/out.test.toml +++ b/acceptance/cache/simple/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + DATABRICKS_CACHE_ENABLED = ["true"] diff --git a/acceptance/cache/simple/output.txt b/acceptance/cache/simple/output.txt index 1c5d30b76c3..726d9c603c8 100644 --- a/acceptance/cache/simple/output.txt +++ b/acceptance/cache/simple/output.txt @@ -1,14 +1,15 @@ === First call in a session is expected to be a cache miss: +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache is enabled for actual use [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted) -[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted) [DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing [DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result === Second call in a session is expected to be a cache hit +[DEBUG_TIMESTAMP] Debug: [Local Cache] cache is enabled for actual use [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] diff --git a/acceptance/cache/simple/test.toml b/acceptance/cache/simple/test.toml index 97c8ceb226b..c9130f113e0 100644 --- a/acceptance/cache/simple/test.toml +++ b/acceptance/cache/simple/test.toml @@ -2,3 +2,6 @@ Cloud=false Local=true RecordRequests = true + +[EnvMatrix] + DATABRICKS_CACHE_ENABLED = ["true"] diff --git a/bundle/bundle.go b/bundle/bundle.go index dca26fc559d..2732f085650 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -13,6 +13,7 @@ import ( "os" "path/filepath" "sync" + "time" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/direct" @@ -50,6 +51,7 @@ type Metrics struct { PythonAddedResourcesCount int64 PythonUpdatedResourcesCount int64 ExecutionTimes []protos.IntMapEntry + LocalCacheMeasurementsMs []protos.IntMapEntry // Local cache measurements stored as milliseconds } // SetBoolValue sets the value of a boolean metric. @@ -70,6 +72,21 @@ func (m *Metrics) AddBoolValue(key string, value bool) { m.BoolValues = append(m.BoolValues, protos.BoolMapEntry{Key: key, Value: value}) } +// SetDurationValue sets the value of a duration metric in milliseconds. +// If the metric does not exist, it is created. +// If the metric exists, it is updated. +// Ensures that the metric is unique. +func (m *Metrics) SetDurationValue(key string, value time.Duration) { + valueMs := value.Milliseconds() + for i, v := range m.LocalCacheMeasurementsMs { + if v.Key == key { + m.LocalCacheMeasurementsMs[i].Value = valueMs + return + } + } + m.LocalCacheMeasurementsMs = append(m.LocalCacheMeasurementsMs, protos.IntMapEntry{Key: key, Value: valueMs}) +} + type Bundle struct { // BundleRootPath is the local path to the root directory of the bundle. // It is set when we instantiate a new bundle instance. diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index e75998f67db..5a54f14d7db 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -3,7 +3,6 @@ package mutator import ( "context" "net/http" - "os" "github.com/databricks/cli/libs/cache" @@ -27,21 +26,18 @@ func PopulateCurrentUser() bundle.Mutator { return &populateCurrentUser{} } -// initializeCache sets up the cache for authorization headers if not already initialized +// initializeCache sets up the cache for authorization headers if not already initialized. +// By default, cache operates in measurement-only mode to gather metrics about potential savings. +// Set DATABRICKS_CACHE_ENABLED=true to enable actual caching. func (m *populateCurrentUser) initializeCache(ctx context.Context, b *bundle.Bundle) { if m.cache != nil { return } - if os.Getenv("DATABRICKS_CACHE_DISABLED") == "true" { - log.Debugf(ctx, "[Local Cache] Local cache is disabled via environment variable DATABRICKS_CACHE_DISABLED=true\n") - return - } - var err error m.cache, err = cache.NewFileCache[*iam.User](ctx, "auth", 30, &b.Metrics) if err != nil { - log.Debugf(ctx, "[Local Cache] Failed to initialize cache dir: %v\n", err) + log.Debugf(ctx, "[Local Cache] Failed to initialize cache: %v\n", err) } } diff --git a/bundle/phases/telemetry.go b/bundle/phases/telemetry.go index f49f55ec845..4584e9fc5e1 100644 --- a/bundle/phases/telemetry.go +++ b/bundle/phases/telemetry.go @@ -175,6 +175,7 @@ func logDeployTelemetry(ctx context.Context, b *bundle.Bundle) { TargetCount: b.Metrics.TargetCount, WorkspaceArtifactPathType: artifactPathType, BoolValues: b.Metrics.BoolValues, + LocalCacheMeasurementsMs: b.Metrics.LocalCacheMeasurementsMs, PythonAddedResourcesCount: b.Metrics.PythonAddedResourcesCount, PythonUpdatedResourcesCount: b.Metrics.PythonUpdatedResourcesCount, PythonResourceLoadersCount: int64(len(experimentalConfig.Python.Resources)), diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 6b08a8af2eb..c4aeb964ca8 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -17,14 +17,16 @@ import ( // Metrics is a local interface for tracking cache telemetry. type Metrics interface { SetBoolValue(key string, value bool) + SetDurationValue(key string, value time.Duration) } // FileCache implements the Cache interface using local disk storage. type FileCache[T any] struct { - baseDir string - expiryMinutes int - mu sync.Mutex - metrics Metrics + baseDir string + expiryMinutes int + mu sync.Mutex + metrics Metrics + measurementMode bool // If true, cache is used only for measurement, not for actual caching } // newFileCacheWithBaseDir creates a new file-based cache that stores data in the specified directory. @@ -112,6 +114,12 @@ func sanitizeVersion(version string) string { // NewFileCache creates a new file-based cache using UserCacheDir() + "databricks" + version + cached component name. // Including the CLI version in the path ensures cache isolation across different CLI versions. +// By default, the cache operates in measurement-only mode (measurementMode=true), which means it will: +// - Check if cached values exist +// - Measure how much time would have been saved +// - Emit metrics about potential savings +// - Always compute the value (never actually use the cache) +// Set DATABRICKS_CACHE_ENABLED=true to enable actual caching. func NewFileCache[T any](ctx context.Context, component string, expiryMinutes int, metrics Metrics) (*FileCache[T], error) { cacheBaseDir, err := getCacheBaseDir() if err != nil { @@ -127,6 +135,17 @@ func NewFileCache[T any](ctx context.Context, component string, expiryMinutes in return nil, err } fc.metrics = metrics + + // By default, cache is in measurement mode (disabled for actual use) + // Explicitly enable with DATABRICKS_CACHE_ENABLED=true + fc.measurementMode = os.Getenv("DATABRICKS_CACHE_ENABLED") != "true" + + if fc.measurementMode { + log.Debugf(ctx, "[Local Cache] cache is in measurement-only mode; set DATABRICKS_CACHE_ENABLED=true to enable caching\n") + } else { + log.Debugf(ctx, "[Local Cache] cache is enabled for actual use\n") + } + return fc, nil } @@ -141,6 +160,8 @@ func (fc *FileCache[T]) addTelemetryMetric(key string) { // GetOrCompute retrieves cached content or computes it using the provided function. // Cache operations fail open: if caching fails, the compute function is still called. +// In measurement mode, the cache checks if values exist and measures potential time savings, +// but always computes and never returns cached values. func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compute func(ctx context.Context) (T, error)) (T, error) { // Convert fingerprint to deterministic hash - this is our cache key cacheKey, err := fingerprintToHash(fingerprint) @@ -156,25 +177,32 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu cachePath := fc.getCachePath(cacheKey) // Try to read from disk cache - if data, found := fc.readFromCache(ctx, cachePath); found { + cachedData, cacheExists := fc.readFromCache(ctx, cachePath) + + // In normal mode: return cached value if found + if cacheExists && !fc.measurementMode { log.Debugf(ctx, "[Local Cache] cache hit\n") fc.addTelemetryMetric("local.cache.hit") - return data, nil + return cachedData, nil } - // Cache miss - acquire lock to compute - fc.mu.Lock() - defer fc.mu.Unlock() - - // Check again after acquiring lock (another goroutine might have computed it) - if data, found := fc.readFromCache(ctx, cachePath); found { - log.Debugf(ctx, "[Local Cache] cache hit after lock\n") + // Record metrics (hit in measurement mode, miss in normal mode) + if cacheExists { + log.Debugf(ctx, "[Local Cache] cache hit\n") fc.addTelemetryMetric("local.cache.hit") - return data, nil + } else { + log.Debugf(ctx, "[Local Cache] cache miss, computing\n") + fc.addTelemetryMetric("local.cache.miss") } - // Compute the value - log.Debugf(ctx, "[Local Cache] cache miss, computing\n") + // In normal mode, acquire lock to serialize writes to the same cache key + if !fc.measurementMode { + fc.mu.Lock() + defer fc.mu.Unlock() + } + + // Compute the value (with timing in measurement mode) + start := time.Now() result, err := compute(ctx) if err != nil { log.Debugf(ctx, "[Local Cache] error while computing: %v\n", err) @@ -182,10 +210,19 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu return result, err } + // Record duration metrics in measurement mode + if fc.measurementMode && fc.metrics != nil { + computeDuration := time.Since(start) + fc.metrics.SetDurationValue("local.cache.compute_duration", computeDuration) + if cacheExists { + fc.metrics.SetDurationValue("local.cache.potential_savings", computeDuration) + } + } + + log.Debugf(ctx, "[Local Cache] computed and stored result\n") + // Write to disk cache (failures are silent - cache write errors don't affect the result) fc.writeToCache(ctx, cachePath, result) - log.Debugf(ctx, "[Local Cache] computed and stored result\n") - fc.addTelemetryMetric("local.cache.miss") return result, nil } diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index 6dc7fba6802..92976178f72 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -167,8 +167,11 @@ func TestFileCacheGetOrComputeConcurrency(t *testing.T) { assert.Equal(t, expectedValue, result) } - // With locking, compute should only be called once even with concurrent requests - assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) + // With locking, writes are serialized but compute may be called multiple times + // since goroutines check cache before acquiring lock + calls := atomic.LoadInt32(&computeCalls) + assert.GreaterOrEqual(t, calls, int32(1), "compute should be called at least once") + assert.LessOrEqual(t, calls, int32(numGoroutines), "compute should not be called more than number of goroutines") } func TestFileCacheCleanupExpiredFiles(t *testing.T) { diff --git a/libs/telemetry/protos/bundle_deploy.go b/libs/telemetry/protos/bundle_deploy.go index b7e3075811a..d613073f9fd 100644 --- a/libs/telemetry/protos/bundle_deploy.go +++ b/libs/telemetry/protos/bundle_deploy.go @@ -80,6 +80,9 @@ type BundleDeployExperimental struct { // Number of resource mutators declared at 'python/mutators' in databricks.yml PythonResourceMutatorsCount int64 `json:"python_resource_mutators_count,omitempty"` + + // Local cache measurements in milliseconds (compute duration, potential savings, etc.) + LocalCacheMeasurementsMs []IntMapEntry `json:"local_cache_measurements_ms,omitempty"` } type BoolMapEntry struct { From 4cd6fd134b6c0ea455006dd654684985ff03e239 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 27 Nov 2025 14:13:10 +0100 Subject: [PATCH 70/87] fix tests --- .../bundle/run/scripts/databricks-cli/test.toml | 2 +- .../telemetry/deploy-compute-type/output.txt | 16 ++++++++++++++++ .../telemetry/deploy-experimental/output.txt | 8 ++++++++ .../deploy-name-prefix/custom/output.txt | 8 ++++++++ .../mode-development/output.txt | 8 ++++++++ .../telemetry/deploy-whl-artifacts/output.txt | 16 ++++++++++++++++ .../bundle/telemetry/deploy/out.telemetry.txt | 11 ++++++++++- acceptance/bundle/telemetry/test.toml | 10 +++++++++- acceptance/bundle/user_agent/test.toml | 2 +- libs/telemetry/protos/bundle_deploy.go | 2 +- 10 files changed, 78 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/run/scripts/databricks-cli/test.toml b/acceptance/bundle/run/scripts/databricks-cli/test.toml index 99d009ab1fe..e42dd928e61 100644 --- a/acceptance/bundle/run/scripts/databricks-cli/test.toml +++ b/acceptance/bundle/run/scripts/databricks-cli/test.toml @@ -2,7 +2,7 @@ RecordRequests = true IncludeRequestHeaders = ["Authorization"] [Env] -DATABRICKS_CACHE_DISABLED = 'true' +DATABRICKS_CACHE_ENABLED = 'false' # "client_id:client_secret" in base64 is Y2xpZW50X2lkOmNsaWVudF9zZWNyZXQ=, expect to # see this in Authorization header diff --git a/acceptance/bundle/telemetry/deploy-compute-type/output.txt b/acceptance/bundle/telemetry/deploy-compute-type/output.txt index f6376bf9dd3..057c39be997 100644 --- a/acceptance/bundle/telemetry/deploy-compute-type/output.txt +++ b/acceptance/bundle/telemetry/deploy-compute-type/output.txt @@ -13,6 +13,14 @@ Deployment complete! >>> cat out.requests.txt [ + { + "key": "local.cache.(redacted) + "value": true + }, + { + "key": "local.cache.(redacted) + "value": true + }, { "key": "experimental.use_legacy_run_as", "value": false @@ -47,6 +55,14 @@ Deployment complete! } ] [ + { + "key": "local.cache.(redacted) + "value": true + }, + { + "key": "local.cache.(redacted) + "value": true + }, { "key": "experimental.use_legacy_run_as", "value": false diff --git a/acceptance/bundle/telemetry/deploy-experimental/output.txt b/acceptance/bundle/telemetry/deploy-experimental/output.txt index 437a3c6f9e9..4bc1fa21121 100644 --- a/acceptance/bundle/telemetry/deploy-experimental/output.txt +++ b/acceptance/bundle/telemetry/deploy-experimental/output.txt @@ -12,6 +12,14 @@ Deployment complete! >>> cat out.requests.txt { "bool_values": [ + { + "key": "local.cache.(redacted) + "value": true + }, + { + "key": "local.cache.(redacted) + "value": true + }, { "key": "experimental.use_legacy_run_as", "value": true diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt b/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt index 567b4282000..e9c44168984 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt @@ -8,6 +8,14 @@ Deployment complete! >>> cat out.requests.txt { "bool_values": [ + { + "key": "local.cache.(redacted) + "value": true + }, + { + "key": "local.cache.(redacted) + "value": true + }, { "key": "experimental.use_legacy_run_as", "value": false diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt index 7a710b9045b..55cbc6c1f04 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt @@ -8,6 +8,14 @@ Deployment complete! >>> cat out.requests.txt { "bool_values": [ + { + "key": "local.cache.(redacted) + "value": true + }, + { + "key": "local.cache.(redacted) + "value": true + }, { "key": "experimental.use_legacy_run_as", "value": false diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt b/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt index 207ee71d24c..aa5a8477e63 100644 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt @@ -16,6 +16,14 @@ Deployment complete! >>> cat out.requests.txt { "bool_values": [ + { + "key": "local.cache.(redacted) + "value": true + }, + { + "key": "local.cache.(redacted) + "value": true + }, { "key": "artifact_build_command_is_set", "value": false @@ -48,6 +56,14 @@ Deployment complete! } { "bool_values": [ + { + "key": "local.cache.(redacted) + "value": true + }, + { + "key": "local.cache.(redacted) + "value": true + }, { "key": "artifact_build_command_is_set", "value": true diff --git a/acceptance/bundle/telemetry/deploy/out.telemetry.txt b/acceptance/bundle/telemetry/deploy/out.telemetry.txt index 57b9b46f855..8b30b76775c 100644 --- a/acceptance/bundle/telemetry/deploy/out.telemetry.txt +++ b/acceptance/bundle/telemetry/deploy/out.telemetry.txt @@ -42,6 +42,14 @@ "lookup_variable_count": 0, "target_count": 1, "bool_values": [ + { + "key": "local.cache.(redacted) + "value": true + }, + { + "key": "local.cache.(redacted) + "value": true + }, { "key": "experimental.use_legacy_run_as", "value": false @@ -76,7 +84,8 @@ } ], "bundle_mode": "TYPE_UNSPECIFIED", - "workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM" + "workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM", + "local_cache_measurements_ms": [] } } } diff --git a/acceptance/bundle/telemetry/test.toml b/acceptance/bundle/telemetry/test.toml index 9bdbed40ebc..a758d49fcba 100644 --- a/acceptance/bundle/telemetry/test.toml +++ b/acceptance/bundle/telemetry/test.toml @@ -2,7 +2,7 @@ RecordRequests = true IncludeRequestHeaders = ["User-Agent"] [Env] -DATABRICKS_CACHE_DISABLED = 'true' +DATABRICKS_CACHE_ENABLED = 'false' [[Repls]] Old = '"execution_time_ms": \d{1,5},' @@ -11,3 +11,11 @@ New = '"execution_time_ms": SMALL_INT,' [[Repls]] Old = '(linux|darwin|windows)' New = '[OS]' + +[[Repls]] +Old = 'local\.cache\.(.*)' +New = 'local.cache.(redacted)' + +[[Repls]] +Old = '"local_cache_measurements_ms": \[[^\]]*\]' +New = '"local_cache_measurements_ms": []' diff --git a/acceptance/bundle/user_agent/test.toml b/acceptance/bundle/user_agent/test.toml index 0934af860a5..9295fedc55e 100644 --- a/acceptance/bundle/user_agent/test.toml +++ b/acceptance/bundle/user_agent/test.toml @@ -3,4 +3,4 @@ Local = true IncludeRequestHeaders = ["User-Agent"] [Env] -DATABRICKS_CACHE_DISABLED = 'true' +DATABRICKS_CACHE_ENABLED = 'false' diff --git a/libs/telemetry/protos/bundle_deploy.go b/libs/telemetry/protos/bundle_deploy.go index d613073f9fd..ab1b3a46de5 100644 --- a/libs/telemetry/protos/bundle_deploy.go +++ b/libs/telemetry/protos/bundle_deploy.go @@ -92,5 +92,5 @@ type BoolMapEntry struct { type IntMapEntry struct { Key string `json:"key,omitempty"` - Value int64 `json:"value,omitempty"` + Value int64 `json:"value"` } From 93d01a9e97a4dee217ca2e05cebad745e82f5b66 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 27 Nov 2025 14:17:56 +0100 Subject: [PATCH 71/87] fix tests --- .../resource_deps/job_tasks/out.telemetry.direct.txt | 2 ++ .../job_tasks/out.telemetry.terraform.txt | 2 ++ acceptance/bundle/resource_deps/job_tasks/output.txt | 12 ------------ 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt index 30f909c79dd..410528fdf3f 100644 --- a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt +++ b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt @@ -2,6 +2,8 @@ experimental.use_legacy_run_as false has_classic_interactive_compute false has_classic_job_compute false has_serverless_compute true +local.cache.attempt true +local.cache.miss true presets_name_prefix_is_set false python_wheel_wrapper_is_set false resref_jobs.tags.* true diff --git a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt index 38ee5032b11..50371a06442 100644 --- a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt +++ b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt @@ -2,6 +2,8 @@ experimental.use_legacy_run_as false has_classic_interactive_compute false has_classic_job_compute false has_serverless_compute true +local.cache.attempt true +local.cache.miss true presets_name_prefix_is_set false python_wheel_wrapper_is_set false resref_jobs.tags.* true diff --git a/acceptance/bundle/resource_deps/job_tasks/output.txt b/acceptance/bundle/resource_deps/job_tasks/output.txt index 64b9a619f74..3ff91361fb4 100644 --- a/acceptance/bundle/resource_deps/job_tasks/output.txt +++ b/acceptance/bundle/resource_deps/job_tasks/output.txt @@ -7,15 +7,3 @@ Updating deployment state... Deployment complete! >>> print_telemetry_bool_values -experimental.use_legacy_run_as false -has_classic_interactive_compute false -has_classic_job_compute false -has_serverless_compute true -local.cache.attempt true -local.cache.miss true -presets_name_prefix_is_set false -python_wheel_wrapper_is_set false -resref_jobs.tags.* true -resreferr_jobs.task true -run_as_set false -skip_artifact_cleanup false From be0374c1eb485308ba20f4e5a9463dd205ef6333 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 27 Nov 2025 14:26:21 +0100 Subject: [PATCH 72/87] fix test --- .../bundle/resource_deps/job_tasks/out.telemetry.direct.txt | 2 +- .../missing_ingestion_definition/out.requests.txt | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt index 410528fdf3f..fb413505b2b 100644 --- a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt +++ b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt @@ -3,7 +3,7 @@ has_classic_interactive_compute false has_classic_job_compute false has_serverless_compute true local.cache.attempt true -local.cache.miss true +local.cache.hit true presets_name_prefix_is_set false python_wheel_wrapper_is_set false resref_jobs.tags.* true diff --git a/acceptance/bundle/resource_deps/missing_ingestion_definition/out.requests.txt b/acceptance/bundle/resource_deps/missing_ingestion_definition/out.requests.txt index 19e52ab2a18..dadad9574ec 100644 --- a/acceptance/bundle/resource_deps/missing_ingestion_definition/out.requests.txt +++ b/acceptance/bundle/resource_deps/missing_ingestion_definition/out.requests.txt @@ -23,6 +23,10 @@ "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files" } } +{ + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} { "method": "GET", "path": "/api/2.0/workspace/get-status", From 88cd440a7edf555f7343a6690e32e655f5090ce3 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 27 Nov 2025 14:36:00 +0100 Subject: [PATCH 73/87] repls --- .../bundle/resource_deps/job_tasks/out.telemetry.direct.txt | 4 ++-- .../resource_deps/job_tasks/out.telemetry.terraform.txt | 4 ++-- acceptance/bundle/resource_deps/resources_var/output.txt | 4 ++-- acceptance/bundle/resource_deps/test.toml | 4 ++++ 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt index fb413505b2b..9c10edcc8b7 100644 --- a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt +++ b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt @@ -2,8 +2,8 @@ experimental.use_legacy_run_as false has_classic_interactive_compute false has_classic_job_compute false has_serverless_compute true -local.cache.attempt true -local.cache.hit true +local.cache.(redacted) +local.cache.(redacted) presets_name_prefix_is_set false python_wheel_wrapper_is_set false resref_jobs.tags.* true diff --git a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt index 50371a06442..ed3c27c9540 100644 --- a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt +++ b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt @@ -2,8 +2,8 @@ experimental.use_legacy_run_as false has_classic_interactive_compute false has_classic_job_compute false has_serverless_compute true -local.cache.attempt true -local.cache.miss true +local.cache.(redacted) +local.cache.(redacted) presets_name_prefix_is_set false python_wheel_wrapper_is_set false resref_jobs.tags.* true diff --git a/acceptance/bundle/resource_deps/resources_var/output.txt b/acceptance/bundle/resource_deps/resources_var/output.txt index cd34790c2ca..51bdc0789c9 100644 --- a/acceptance/bundle/resource_deps/resources_var/output.txt +++ b/acceptance/bundle/resource_deps/resources_var/output.txt @@ -40,8 +40,8 @@ experimental.use_legacy_run_as false has_classic_interactive_compute false has_classic_job_compute false has_serverless_compute false -local.cache.attempt true -local.cache.hit true +local.cache.(redacted) +local.cache.(redacted) presets_name_prefix_is_set true python_wheel_wrapper_is_set false resref_volumes.catalog_name true diff --git a/acceptance/bundle/resource_deps/test.toml b/acceptance/bundle/resource_deps/test.toml index a2b2d9fc33f..78ef97636e9 100644 --- a/acceptance/bundle/resource_deps/test.toml +++ b/acceptance/bundle/resource_deps/test.toml @@ -5,3 +5,7 @@ Ignore = [ ".databricks", ".gitignore", ] + +[[Repls]] +Old = 'local\.cache\.(.*)' +New = 'local.cache.(redacted)' From ecc4e9c88f6050a737042bbc9844504a856c84f4 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 27 Nov 2025 16:51:06 +0100 Subject: [PATCH 74/87] simplify --- .../bundle/telemetry/deploy/out.telemetry.txt | 2 +- acceptance/bundle/telemetry/test.toml | 2 +- acceptance/cache/clear/out.test.toml | 1 - acceptance/cache/clear/output.txt | 3 - acceptance/cache/clear/test.toml | 4 +- acceptance/cache/simple/out.test.toml | 1 - acceptance/cache/simple/output.txt | 2 - acceptance/cache/simple/test.toml | 4 +- libs/cache/file_cache.go | 63 +++++++------------ libs/cache/file_cache_expiry_test.go | 17 +++++ libs/cache/file_cache_test.go | 3 + 11 files changed, 47 insertions(+), 55 deletions(-) diff --git a/acceptance/bundle/telemetry/deploy/out.telemetry.txt b/acceptance/bundle/telemetry/deploy/out.telemetry.txt index 8b30b76775c..b932d24e27f 100644 --- a/acceptance/bundle/telemetry/deploy/out.telemetry.txt +++ b/acceptance/bundle/telemetry/deploy/out.telemetry.txt @@ -85,7 +85,7 @@ ], "bundle_mode": "TYPE_UNSPECIFIED", "workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM", - "local_cache_measurements_ms": [] + "local_cache_measurements_ms": [...redacted...] } } } diff --git a/acceptance/bundle/telemetry/test.toml b/acceptance/bundle/telemetry/test.toml index a758d49fcba..4c525096880 100644 --- a/acceptance/bundle/telemetry/test.toml +++ b/acceptance/bundle/telemetry/test.toml @@ -18,4 +18,4 @@ New = 'local.cache.(redacted)' [[Repls]] Old = '"local_cache_measurements_ms": \[[^\]]*\]' -New = '"local_cache_measurements_ms": []' +New = '"local_cache_measurements_ms": [...redacted...]' diff --git a/acceptance/cache/clear/out.test.toml b/acceptance/cache/clear/out.test.toml index 7af6fdc0a51..d560f1de043 100644 --- a/acceptance/cache/clear/out.test.toml +++ b/acceptance/cache/clear/out.test.toml @@ -3,4 +3,3 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] - DATABRICKS_CACHE_ENABLED = ["true"] diff --git a/acceptance/cache/clear/output.txt b/acceptance/cache/clear/output.txt index 8999e205531..749cf917494 100644 --- a/acceptance/cache/clear/output.txt +++ b/acceptance/cache/clear/output.txt @@ -1,6 +1,5 @@ === First call in a session is expected to be a cache miss: -[DEBUG_TIMESTAMP] Debug: [Local Cache] cache is enabled for actual use [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] @@ -9,7 +8,6 @@ [DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result === Second call in a session is expected to be a cache hit -[DEBUG_TIMESTAMP] Debug: [Local Cache] cache is enabled for actual use [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] @@ -19,7 +17,6 @@ Cache cleared successfully from [TEST_TMP_DIR]/.cache === First call after a clear is expected to be a cache miss: -[DEBUG_TIMESTAMP] Debug: [Local Cache] cache is enabled for actual use [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] diff --git a/acceptance/cache/clear/test.toml b/acceptance/cache/clear/test.toml index 746f844de76..6cad75d0f02 100644 --- a/acceptance/cache/clear/test.toml +++ b/acceptance/cache/clear/test.toml @@ -1,5 +1,5 @@ Cloud=false Local=true -[EnvMatrix] - DATABRICKS_CACHE_ENABLED = ["true"] +[Env] +DATABRICKS_CACHE_ENABLED = 'true' diff --git a/acceptance/cache/simple/out.test.toml b/acceptance/cache/simple/out.test.toml index 7af6fdc0a51..d560f1de043 100644 --- a/acceptance/cache/simple/out.test.toml +++ b/acceptance/cache/simple/out.test.toml @@ -3,4 +3,3 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] - DATABRICKS_CACHE_ENABLED = ["true"] diff --git a/acceptance/cache/simple/output.txt b/acceptance/cache/simple/output.txt index 726d9c603c8..cf3fce0dea1 100644 --- a/acceptance/cache/simple/output.txt +++ b/acceptance/cache/simple/output.txt @@ -1,6 +1,5 @@ === First call in a session is expected to be a cache miss: -[DEBUG_TIMESTAMP] Debug: [Local Cache] cache is enabled for actual use [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] @@ -9,7 +8,6 @@ [DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result === Second call in a session is expected to be a cache hit -[DEBUG_TIMESTAMP] Debug: [Local Cache] cache is enabled for actual use [DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] diff --git a/acceptance/cache/simple/test.toml b/acceptance/cache/simple/test.toml index c9130f113e0..332d8256d02 100644 --- a/acceptance/cache/simple/test.toml +++ b/acceptance/cache/simple/test.toml @@ -3,5 +3,5 @@ Local=true RecordRequests = true -[EnvMatrix] - DATABRICKS_CACHE_ENABLED = ["true"] +[Env] +DATABRICKS_CACHE_ENABLED = 'true' diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index c4aeb964ca8..fa882ecb6df 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -22,11 +22,11 @@ type Metrics interface { // FileCache implements the Cache interface using local disk storage. type FileCache[T any] struct { - baseDir string - expiryMinutes int - mu sync.Mutex - metrics Metrics - measurementMode bool // If true, cache is used only for measurement, not for actual caching + baseDir string + expiryMinutes int + mu sync.Mutex + metrics Metrics + cacheEnabled bool // If true, cached values are returned; if false, cache is only used for measurement } // newFileCacheWithBaseDir creates a new file-based cache that stores data in the specified directory. @@ -114,7 +114,7 @@ func sanitizeVersion(version string) string { // NewFileCache creates a new file-based cache using UserCacheDir() + "databricks" + version + cached component name. // Including the CLI version in the path ensures cache isolation across different CLI versions. -// By default, the cache operates in measurement-only mode (measurementMode=true), which means it will: +// By default, the cache operates in measurement-only mode (cacheEnabled=false), which means it will: // - Check if cached values exist // - Measure how much time would have been saved // - Emit metrics about potential savings @@ -136,16 +136,8 @@ func NewFileCache[T any](ctx context.Context, component string, expiryMinutes in } fc.metrics = metrics - // By default, cache is in measurement mode (disabled for actual use) - // Explicitly enable with DATABRICKS_CACHE_ENABLED=true - fc.measurementMode = os.Getenv("DATABRICKS_CACHE_ENABLED") != "true" - - if fc.measurementMode { - log.Debugf(ctx, "[Local Cache] cache is in measurement-only mode; set DATABRICKS_CACHE_ENABLED=true to enable caching\n") - } else { - log.Debugf(ctx, "[Local Cache] cache is enabled for actual use\n") - } - + // Check if cache is enabled; default is false (measurement-only mode) + fc.cacheEnabled = os.Getenv("DATABRICKS_CACHE_ENABLED") == "true" return fc, nil } @@ -160,7 +152,7 @@ func (fc *FileCache[T]) addTelemetryMetric(key string) { // GetOrCompute retrieves cached content or computes it using the provided function. // Cache operations fail open: if caching fails, the compute function is still called. -// In measurement mode, the cache checks if values exist and measures potential time savings, +// When cacheEnabled is false, the cache checks if values exist and measures potential time savings, // but always computes and never returns cached values. func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compute func(ctx context.Context) (T, error)) (T, error) { // Convert fingerprint to deterministic hash - this is our cache key @@ -179,29 +171,25 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu // Try to read from disk cache cachedData, cacheExists := fc.readFromCache(ctx, cachePath) - // In normal mode: return cached value if found - if cacheExists && !fc.measurementMode { - log.Debugf(ctx, "[Local Cache] cache hit\n") - fc.addTelemetryMetric("local.cache.hit") - return cachedData, nil - } - - // Record metrics (hit in measurement mode, miss in normal mode) + // Record metrics if cacheExists { log.Debugf(ctx, "[Local Cache] cache hit\n") fc.addTelemetryMetric("local.cache.hit") + + // If cache is enabled, return the cached value + if fc.cacheEnabled { + return cachedData, nil + } } else { log.Debugf(ctx, "[Local Cache] cache miss, computing\n") fc.addTelemetryMetric("local.cache.miss") } - // In normal mode, acquire lock to serialize writes to the same cache key - if !fc.measurementMode { - fc.mu.Lock() - defer fc.mu.Unlock() - } + // Acquire lock to prevent concurrent computations and writes for the same cache key + fc.mu.Lock() + defer fc.mu.Unlock() - // Compute the value (with timing in measurement mode) + // Compute the value and measure timing start := time.Now() result, err := compute(ctx) if err != nil { @@ -210,13 +198,10 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu return result, err } - // Record duration metrics in measurement mode - if fc.measurementMode && fc.metrics != nil { + // Record duration metrics + if fc.metrics != nil { computeDuration := time.Since(start) fc.metrics.SetDurationValue("local.cache.compute_duration", computeDuration) - if cacheExists { - fc.metrics.SetDurationValue("local.cache.potential_savings", computeDuration) - } } log.Debugf(ctx, "[Local Cache] computed and stored result\n") @@ -271,12 +256,6 @@ func (fc *FileCache[T]) writeToCache(ctx context.Context, cachePath string, data return // Silently fail on serialization errors } - // Ensure directory exists - if err := os.MkdirAll(filepath.Dir(cachePath), 0o700); err != nil { - log.Debugf(ctx, "[Local Cache] failed to create directory: %v\n", err) - return - } - // Write to cache file - the mtime will be used to track expiry err = os.WriteFile(cachePath, serializedData, 0o600) if err != nil { diff --git a/libs/cache/file_cache_expiry_test.go b/libs/cache/file_cache_expiry_test.go index 847cdd0e79a..02da219f177 100644 --- a/libs/cache/file_cache_expiry_test.go +++ b/libs/cache/file_cache_expiry_test.go @@ -20,6 +20,9 @@ func TestFileCacheExpiryBehavior(t *testing.T) { cache, err := newFileCacheWithBaseDir[string](ctx, tempDir, 1) require.NoError(t, err) + // Enable cache for this test (default is measurement-only mode) + cache.cacheEnabled = true + fingerprint := struct { Key string `json:"key"` }{ @@ -48,6 +51,20 @@ func TestFileCacheExpiryBehavior(t *testing.T) { require.NoError(t, err) age := time.Since(info.ModTime()) assert.Less(t, age, 10*time.Second, "File should have been created recently") + + // Make the file expired by backdating its mtime to 2 minutes ago (older than 1 minute expiry) + expiredTime := time.Now().Add(-2 * time.Minute) + require.NoError(t, os.Chtimes(cacheFiles[0], expiredTime, expiredTime)) + + // Verify GetOrCompute treats it as a cache miss and recomputes + callCount := 0 + result, err = cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + callCount++ + return "recomputed-value", nil + }) + require.NoError(t, err) + assert.Equal(t, "recomputed-value", result, "Should return newly computed value, not expired cache") + assert.Equal(t, 1, callCount, "Should have called compute function once due to cache expiry") } // TestReadFromCacheRespectsExpiry tests that readFromCache returns false for expired entries based on mtime diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index 92976178f72..70141289ea0 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -75,6 +75,9 @@ func TestFileCacheGetOrCompute(t *testing.T) { cache, err := newFileCacheWithBaseDir[string](ctx, tempDir, 60) // 1 hour for tests require.NoError(t, err) + // Enable cache for this test (default is measurement-only mode) + cache.cacheEnabled = true + fingerprint := struct { Key string `json:"key"` Value int `json:"value"` From 4fad903c45985bbd8bf049a1963c0329164b9883 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 1 Dec 2025 12:45:14 +0100 Subject: [PATCH 75/87] addressed feedback + refactor --- acceptance/acceptance_test.go | 5 +- acceptance/cache/clear/output.txt | 3 - acceptance/cache/clear/test.toml | 13 +- acceptance/cache/simple/output.txt | 2 - acceptance/cache/simple/test.toml | 13 +- acceptance/internal/config.go | 2 + .../config/mutator/populate_current_user.go | 37 +-- bundle/fingerprint.go | 34 +++ cmd/cache/cache.go | 16 +- libs/cache/cache.go | 18 -- libs/cache/file_cache.go | 97 ++++--- libs/cache/file_cache_clear.go | 20 +- libs/cache/file_cache_env_test.go | 186 ++++++++++++++ libs/cache/file_cache_test.go | 240 ++++++++++++++++-- libs/cache/fingerprint.go | 22 ++ libs/cache/fingerprint_test.go | 33 +++ libs/cache/noop_file_cache.go | 9 + 17 files changed, 619 insertions(+), 131 deletions(-) create mode 100644 bundle/fingerprint.go create mode 100644 libs/cache/file_cache_env_test.go create mode 100644 libs/cache/fingerprint.go create mode 100644 libs/cache/fingerprint_test.go create mode 100644 libs/cache/noop_file_cache.go diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 9880aca9d39..8dac7458bc9 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -629,9 +629,8 @@ func runTest(t *testing.T, } // Set unique cache folder for this test to avoid race conditions between parallel tests - userCacheDir, err := os.UserCacheDir() - require.NoError(t, err) - uniqueCacheDir := filepath.Join(userCacheDir, strings.ReplaceAll(dir, string(os.PathSeparator), "--")) + // Use test temp directory to avoid polluting user's cache + uniqueCacheDir := filepath.Join(tmpDir, ".cache") cmd.Env = append(cmd.Env, "DATABRICKS_CACHE_DIR="+uniqueCacheDir) for _, key := range utils.SortedKeys(config.Env) { diff --git a/acceptance/cache/clear/output.txt b/acceptance/cache/clear/output.txt index 749cf917494..19ffef35021 100644 --- a/acceptance/cache/clear/output.txt +++ b/acceptance/cache/clear/output.txt @@ -1,6 +1,5 @@ === First call in a session is expected to be a cache miss: -[DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted) @@ -8,7 +7,6 @@ [DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result === Second call in a session is expected to be a cache hit -[DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit @@ -17,7 +15,6 @@ Cache cleared successfully from [TEST_TMP_DIR]/.cache === First call after a clear is expected to be a cache miss: -[DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted) diff --git a/acceptance/cache/clear/test.toml b/acceptance/cache/clear/test.toml index 6cad75d0f02..0b1b2fe5e7e 100644 --- a/acceptance/cache/clear/test.toml +++ b/acceptance/cache/clear/test.toml @@ -1,5 +1,14 @@ -Cloud=false -Local=true +Cloud = false +Local = true [Env] DATABRICKS_CACHE_ENABLED = 'true' + +# Redact structured logging fields from debug output +[[Repls]] +Old = ' pid=[0-9]+' +New = '' + +[[Repls]] +Old = ' mutator=[A-Za-z]+' +New = '' diff --git a/acceptance/cache/simple/output.txt b/acceptance/cache/simple/output.txt index cf3fce0dea1..40df5673b5f 100644 --- a/acceptance/cache/simple/output.txt +++ b/acceptance/cache/simple/output.txt @@ -1,6 +1,5 @@ === First call in a session is expected to be a cache miss: -[DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted) @@ -8,7 +7,6 @@ [DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result === Second call in a session is expected to be a cache hit -[DEBUG_TIMESTAMP] Debug: [Local Cache] found authorization header with length: 45 [DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit diff --git a/acceptance/cache/simple/test.toml b/acceptance/cache/simple/test.toml index 332d8256d02..07f6a81177e 100644 --- a/acceptance/cache/simple/test.toml +++ b/acceptance/cache/simple/test.toml @@ -1,7 +1,16 @@ -Cloud=false -Local=true +Cloud = false +Local = true RecordRequests = true [Env] DATABRICKS_CACHE_ENABLED = 'true' + +# Redact structured logging fields from debug output +[[Repls]] +Old = ' pid=[0-9]+' +New = '' + +[[Repls]] +Old = ' mutator=[A-Za-z]+' +New = '' diff --git a/acceptance/internal/config.go b/acceptance/internal/config.go index 5f3d0309478..2129fa55737 100644 --- a/acceptance/internal/config.go +++ b/acceptance/internal/config.go @@ -213,6 +213,8 @@ func LoadConfig(t *testing.T, dir string) (TestConfig, string) { } } + // Always ignore .cache directory (used by local cache) + result.Ignore = append(result.Ignore, ".cache") result.CompiledIgnoreObject = ignore.CompileIgnoreLines(result.Ignore...) return result, strings.Join(configs, ", ") diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index 5a54f14d7db..74f13ea5381 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -2,7 +2,6 @@ package mutator import ( "context" - "net/http" "github.com/databricks/cli/libs/cache" @@ -13,7 +12,6 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/iamutil" "github.com/databricks/cli/libs/tags" - "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/iam" ) @@ -30,15 +28,7 @@ func PopulateCurrentUser() bundle.Mutator { // By default, cache operates in measurement-only mode to gather metrics about potential savings. // Set DATABRICKS_CACHE_ENABLED=true to enable actual caching. func (m *populateCurrentUser) initializeCache(ctx context.Context, b *bundle.Bundle) { - if m.cache != nil { - return - } - - var err error - m.cache, err = cache.NewFileCache[*iam.User](ctx, "auth", 30, &b.Metrics) - if err != nil { - log.Debugf(ctx, "[Local Cache] Failed to initialize cache: %v\n", err) - } + m.cache = cache.NewCache[*iam.User](ctx, "auth", 30, &b.Metrics) } func (m *populateCurrentUser) Name() string { @@ -52,23 +42,18 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. m.initializeCache(ctx, b) w := b.WorkspaceClient() - fingerprint := struct { - authHeader string - }{ - authHeader: m.getAuthorizationHeader(ctx, w), - } - var me *iam.User var err error - if m.cache != nil && fingerprint.authHeader != "" { - log.Debugf(ctx, "[Local Cache] local cache is enabled\n") + fingerprint := b.GetUserFingerprint(ctx) + if !fingerprint.IsEmpty() { + log.Debugf(ctx, "[Local Cache] local cache is enabled") me, err = m.cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (*iam.User, error) { currentUser, err := w.CurrentUser.Me(ctx) return currentUser, err }) } else { - log.Debugf(ctx, "[Local Cache] local cache is disabled\n") + log.Debugf(ctx, "[Local Cache] local cache is disabled") me, err = w.CurrentUser.Me(ctx) } @@ -91,15 +76,3 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. return nil } - -func (m *populateCurrentUser) getAuthorizationHeader(ctx context.Context, w *databricks.WorkspaceClient) string { - // Create a dummy request to extract the Authorization header - req := &http.Request{Header: http.Header{}} - if err := w.Config.Authenticate(req); err != nil { - return "" - } - - authHeader := req.Header.Get("Authorization") - log.Debugf(ctx, "[Local Cache] found authorization header with length: %d\n", len(authHeader)) - return authHeader -} diff --git a/bundle/fingerprint.go b/bundle/fingerprint.go new file mode 100644 index 00000000000..526547b0ab5 --- /dev/null +++ b/bundle/fingerprint.go @@ -0,0 +1,34 @@ +package bundle + +import ( + "context" + "net/http" +) + +type UserFingerprint struct { + Host string `json:"host"` + AuthHeader string `json:"auth_header"` +} + +func (f *UserFingerprint) IsEmpty() bool { + return f.Host == "" && f.AuthHeader == "" +} + +func (b *Bundle) GetUserFingerprint(ctx context.Context) UserFingerprint { + return UserFingerprint{ + Host: b.WorkspaceClient().Config.Host, + AuthHeader: b.getAuthorizationHeader(), + } +} + +// getAuthorizationHeader extracts the Authorization header from the workspace client configuration. +// If it fails to authenticate, it returns an empty string. +func (b *Bundle) getAuthorizationHeader() string { + // Create a dummy request to extract the Authorization header + req := &http.Request{Header: http.Header{}} + if err := b.WorkspaceClient().Config.Authenticate(req); err != nil { + return "" + } + + return req.Header.Get("Authorization") +} diff --git a/cmd/cache/cache.go b/cmd/cache/cache.go index 18c213ff323..88dec62acc6 100644 --- a/cmd/cache/cache.go +++ b/cmd/cache/cache.go @@ -20,10 +20,22 @@ func newClearCommand() *cobra.Command { cmd := &cobra.Command{ Use: "clear", Short: "Clear all local cache files", - Long: "Remove all cached files stored locally by the Databricks CLI", + Long: `Remove all cached files stored locally by the Databricks CLI. + +This clears the cache for all CLI versions, not just the current version. +The cache directory is typically located at: + - Linux/macOS: ~/.cache/databricks/ + - Windows: %LOCALAPPDATA%\databricks\ + +You can override this with the DATABRICKS_CACHE_DIR environment variable.`, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - return cache.ClearFileCache(ctx) + cachePath, err := cache.ClearFileCache(ctx) + if err != nil { + return err + } + cmd.Printf("Cache cleared successfully from %s\n", cachePath) + return nil }, } return cmd diff --git a/libs/cache/cache.go b/libs/cache/cache.go index 2edfc50f706..5eec234666a 100644 --- a/libs/cache/cache.go +++ b/libs/cache/cache.go @@ -2,10 +2,6 @@ package cache import ( "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" ) // Cache provides an abstract interface for caching content to local disk. @@ -21,17 +17,3 @@ type Cache[T any] interface { // Returns an error only if the compute function fails. GetOrCompute(ctx context.Context, fingerprint any, compute func(ctx context.Context) (T, error)) (T, error) } - -// fingerprintToHash converts any struct to a deterministic string representation for use as a cache key. -func fingerprintToHash(fingerprint any) (string, error) { - // Marshal map - json.Marshal sorts map keys alphabetically - data, err := json.Marshal(fingerprint) - if err != nil { - return "", fmt.Errorf("failed to marshal normalized fingerprint: %w", err) - } - - // Hash for consistent, reasonably-sized key. - // hash[:] converts the [32]byte array returned by Sum256 to a []byte slice. - hash := sha256.Sum256(data) - return hex.EncodeToString(hash[:]), nil -} diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index fa882ecb6df..f4eb7bae20d 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -11,6 +11,7 @@ import ( "time" "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" ) @@ -46,13 +47,16 @@ func newFileCacheWithBaseDir[T any](ctx context.Context, baseDir string, expiryM return fc, nil } +// isExpired checks if a file with the given modification time has expired. +func (fc *FileCache[T]) isExpired(modTime time.Time) bool { + expiryThreshold := time.Now().Add(-time.Duration(fc.expiryMinutes) * time.Minute) + return modTime.Before(expiryThreshold) +} + // cleanupExpiredFiles removes expired cache files from disk based on file modification time. // This runs synchronously once when the cache is created. // Files older than expiryMinutes are deleted. func (fc *FileCache[T]) cleanupExpiredFiles(ctx context.Context) { - now := time.Now() - expiryDuration := time.Duration(fc.expiryMinutes) * time.Minute - err := filepath.Walk(fc.baseDir, func(path string, info os.FileInfo, err error) error { if err != nil { // Log walk errors but continue cleanup @@ -64,31 +68,36 @@ func (fc *FileCache[T]) cleanupExpiredFiles(ctx context.Context) { return nil } + // Remove any leftover .tmp files (from failed atomic writes) + if filepath.Ext(info.Name()) == ".tmp" { + _ = os.Remove(path) + return nil + } + // Only process .json cache files if filepath.Ext(info.Name()) != ".json" { return nil } // Check if file is expired based on modification time - age := now.Sub(info.ModTime()) - if age > expiryDuration { + if fc.isExpired(info.ModTime()) { if err := os.Remove(path); err != nil { - log.Debugf(ctx, "[Local Cache] cleanup: failed to remove expired file %s: %v", path, err) + log.Tracef(ctx, "[Local Cache] cleanup: failed to remove expired file %s: %v", path, err) } else { - log.Debugf(ctx, "[Local Cache] cleanup: removed expired file %s (age: %v)", path, age) + log.Tracef(ctx, "[Local Cache] cleanup: removed expired file %s", path) } } return nil }) if err != nil { - log.Warnf(ctx, "[Local Cache] cleanup: failed to walk cache directory: %v", err) + log.Debugf(ctx, "[Local Cache] cleanup: failed to walk cache directory: %v", err) } } -func getCacheBaseDir() (string, error) { +func getCacheBaseDir(ctx context.Context) (string, error) { // Check if user has configured a custom cache directory - if customCacheDir := os.Getenv("DATABRICKS_CACHE_DIR"); customCacheDir != "" { + if customCacheDir := env.Get(ctx, "DATABRICKS_CACHE_DIR"); customCacheDir != "" { return customCacheDir, nil } @@ -112,7 +121,7 @@ func sanitizeVersion(version string) string { return version } -// NewFileCache creates a new file-based cache using UserCacheDir() + "databricks" + version + cached component name. +// NewCache creates a new file-based cache using UserCacheDir() + "databricks" + version + cached component name. // Including the CLI version in the path ensures cache isolation across different CLI versions. // By default, the cache operates in measurement-only mode (cacheEnabled=false), which means it will: // - Check if cached values exist @@ -120,10 +129,10 @@ func sanitizeVersion(version string) string { // - Emit metrics about potential savings // - Always compute the value (never actually use the cache) // Set DATABRICKS_CACHE_ENABLED=true to enable actual caching. -func NewFileCache[T any](ctx context.Context, component string, expiryMinutes int, metrics Metrics) (*FileCache[T], error) { - cacheBaseDir, err := getCacheBaseDir() +func NewCache[T any](ctx context.Context, component string, expiryMinutes int, metrics Metrics) Cache[T] { + cacheBaseDir, err := getCacheBaseDir(ctx) if err != nil { - return nil, err + return &NoopFileCache[T]{} } // Include CLI version in cache path to avoid issues across versions @@ -132,18 +141,15 @@ func NewFileCache[T any](ctx context.Context, component string, expiryMinutes in baseDir := filepath.Join(cacheBaseDir, version, component) fc, err := newFileCacheWithBaseDir[T](ctx, baseDir, expiryMinutes) if err != nil { - return nil, err + return &NoopFileCache[T]{} } fc.metrics = metrics // Check if cache is enabled; default is false (measurement-only mode) - fc.cacheEnabled = os.Getenv("DATABRICKS_CACHE_ENABLED") == "true" - return fc, nil + fc.cacheEnabled = env.Get(ctx, "DATABRICKS_CACHE_ENABLED") == "true" + return fc } -// Cache files are stored as JSON directly without metadata wrapper. -// Expiry is tracked using file modification time, not stored in the file itself. - func (fc *FileCache[T]) addTelemetryMetric(key string) { if fc.metrics != nil { fc.metrics.SetBoolValue(key, true) @@ -159,11 +165,11 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu cacheKey, err := fingerprintToHash(fingerprint) if err != nil { // Fail open: if we can't generate cache key, just compute directly - log.Debugf(ctx, "[Local Cache] failed to generate cache key, computing without cache: %v\n", err) + log.Debugf(ctx, "[Local Cache] failed to generate cache key, computing without cache: %v", err) return compute(ctx) } - log.Debugf(ctx, "[Local Cache] using cache key: %s\n", cacheKey) + log.Debugf(ctx, "[Local Cache] using cache key: %s", cacheKey) fc.addTelemetryMetric("local.cache.attempt") cachePath := fc.getCachePath(cacheKey) @@ -173,7 +179,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu // Record metrics if cacheExists { - log.Debugf(ctx, "[Local Cache] cache hit\n") + log.Debugf(ctx, "[Local Cache] cache hit") fc.addTelemetryMetric("local.cache.hit") // If cache is enabled, return the cached value @@ -181,7 +187,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu return cachedData, nil } } else { - log.Debugf(ctx, "[Local Cache] cache miss, computing\n") + log.Debugf(ctx, "[Local Cache] cache miss, computing") fc.addTelemetryMetric("local.cache.miss") } @@ -193,7 +199,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu start := time.Now() result, err := compute(ctx) if err != nil { - log.Debugf(ctx, "[Local Cache] error while computing: %v\n", err) + log.Debugf(ctx, "[Local Cache] error while computing: %v", err) fc.addTelemetryMetric("local.cache.error") return result, err } @@ -204,7 +210,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu fc.metrics.SetDurationValue("local.cache.compute_duration", computeDuration) } - log.Debugf(ctx, "[Local Cache] computed and stored result\n") + log.Debugf(ctx, "[Local Cache] computed and stored result") // Write to disk cache (failures are silent - cache write errors don't affect the result) fc.writeToCache(ctx, cachePath, result) @@ -220,46 +226,61 @@ func (fc *FileCache[T]) readFromCache(ctx context.Context, cachePath string) (T, // Check file modification time for expiry info, err := os.Stat(cachePath) if err != nil { - log.Debugf(ctx, "[Local Cache] failed to stat cache file: %v\n", err) + log.Debugf(ctx, "[Local Cache] failed to stat cache file: %v", err) return zero, false } - age := time.Since(info.ModTime()) - expiryDuration := time.Duration(fc.expiryMinutes) * time.Minute - if age > expiryDuration { + if fc.isExpired(info.ModTime()) { return zero, false } // Read and deserialize the data data, err := os.ReadFile(cachePath) if err != nil { - log.Debugf(ctx, "[Local Cache] failed to read cache file: %v\n", err) + log.Debugf(ctx, "[Local Cache] failed to read cache file: %v", err) return zero, false } var result T if err := json.Unmarshal(data, &result); err != nil { - log.Debugf(ctx, "[Local Cache] failed to deserialize data: %v\n", err) + log.Debugf(ctx, "[Local Cache] failed to deserialize data: %v", err) return zero, false } return result, true } -// writeToCache serializes and writes data to the cache file. -// Expiry is tracked by file modification time, not stored in the file. +// writeToCache serializes and writes data to the cache file atomically. +// Uses atomic write: writes to temp file first, then renames to actual cache file. func (fc *FileCache[T]) writeToCache(ctx context.Context, cachePath string, data any) { // Serialize the data directly serializedData, err := json.Marshal(data) if err != nil { - log.Debugf(ctx, "[Local Cache] failed to serialize data: %v\n", err) + log.Debugf(ctx, "[Local Cache] failed to serialize data: %v", err) return // Silently fail on serialization errors } - // Write to cache file - the mtime will be used to track expiry - err = os.WriteFile(cachePath, serializedData, 0o600) + // Create temporary file in the same directory for atomic operation + tempFile, err := os.CreateTemp(fc.baseDir, ".cache-*.tmp") if err != nil { - log.Debugf(ctx, "[Local Cache] failed to write to cache file: %v\n", err) + log.Debugf(ctx, "[Local Cache] failed to create temp cache file: %v", err) + return + } + tempPath := tempFile.Name() + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempPath) // Clean up temp file if still exists + }() + + // Write data to temp file + if _, err := tempFile.Write(serializedData); err != nil { + log.Debugf(ctx, "[Local Cache] failed to write to temp cache file: %v", err) + return + } + + // Atomically rename temp file to actual cache file + if err := os.Rename(tempPath, cachePath); err != nil { + log.Debugf(ctx, "[Local Cache] failed to rename temp cache file: %v", err) } } diff --git a/libs/cache/file_cache_clear.go b/libs/cache/file_cache_clear.go index c22dca00a4c..01ca6d15244 100644 --- a/libs/cache/file_cache_clear.go +++ b/libs/cache/file_cache_clear.go @@ -3,22 +3,24 @@ package cache import ( "context" "os" - - "github.com/databricks/cli/libs/cmdio" ) -func ClearFileCache(ctx context.Context) error { - databricksCacheDir, err := getCacheBaseDir() +// ClearFileCache removes all cached files from the Databricks cache directory. +// This clears the cache for ALL CLI versions, not just the current version. +// The cache is organized as: /// +// This function removes the entire directory. +// Returns the path of the cleared directory on success. +func ClearFileCache(ctx context.Context) (string, error) { + databricksCacheDir, err := getCacheBaseDir(ctx) if err != nil { - return err + return "", err } - // Remove the entire databricks cache directory + // Remove the entire databricks cache directory (all versions) err = os.RemoveAll(databricksCacheDir) if err != nil { - return err + return "", err } - cmdio.LogString(ctx, "Cache cleared successfully from "+databricksCacheDir) - return nil + return databricksCacheDir, nil } diff --git a/libs/cache/file_cache_env_test.go b/libs/cache/file_cache_env_test.go new file mode 100644 index 00000000000..709746726f8 --- /dev/null +++ b/libs/cache/file_cache_env_test.go @@ -0,0 +1,186 @@ +package cache + +import ( + "context" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCacheEnabledEnvVar(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + + tests := []struct { + name string + envValue string + expectCached bool + }{ + { + name: "cache enabled with 'true'", + envValue: "true", + expectCached: true, + }, + { + name: "cache disabled with 'false'", + envValue: "false", + expectCached: false, + }, + { + name: "cache disabled when empty", + envValue: "", + expectCached: false, + }, + { + name: "cache disabled with invalid value", + envValue: "yes", + expectCached: false, + }, + { + name: "cache disabled with '1'", + envValue: "1", + expectCached: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Set up environment + if tt.envValue != "" { + t.Setenv("DATABRICKS_CACHE_ENABLED", tt.envValue) + } else { + os.Unsetenv("DATABRICKS_CACHE_ENABLED") + } + + // Note: We can't reset sync.Once, so logging will only happen in the first test + // This is acceptable as we're testing behavior, not logging + + // Create a unique subdirectory for this test + testDir := filepath.Join(tempDir, tt.name) + cache, err := newFileCacheWithBaseDir[string](ctx, testDir, 60) + require.NoError(t, err) + + // Set cacheEnabled based on env var (simulate NewFileCache behavior) + cache.cacheEnabled = env.Get(ctx, "DATABRICKS_CACHE_ENABLED") == "true" + + fingerprint := struct { + Key string `json:"key"` + }{ + Key: "test-key", + } + + // First call - should always compute + var computeCalls int32 + result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + atomic.AddInt32(&computeCalls, 1) + return "computed-value", nil + }) + require.NoError(t, err) + assert.Equal(t, "computed-value", result) + assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) + + // Second call - should use cache only if enabled + result2, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + atomic.AddInt32(&computeCalls, 1) + return "should-not-be-called", nil + }) + require.NoError(t, err) + + if tt.expectCached { + // Cache enabled - should return cached value + assert.Equal(t, "computed-value", result2) + assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls), "Should not recompute when cache is enabled") + } else { + // Cache disabled - should recompute + assert.Equal(t, "should-not-be-called", result2) + assert.Equal(t, int32(2), atomic.LoadInt32(&computeCalls), "Should recompute when cache is disabled") + } + }) + } +} + +func TestCacheDirEnvVar(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + + t.Run("uses DATABRICKS_CACHE_DIR when set", func(t *testing.T) { + customCacheDir := filepath.Join(tempDir, "custom-cache") + t.Setenv("DATABRICKS_CACHE_DIR", customCacheDir) + + cache := NewCache[string](ctx, "test-component", 60, nil) + fc, ok := cache.(*FileCache[string]) + require.True(t, ok) + + // Verify the cache directory is under the custom path + assert.Contains(t, fc.baseDir, customCacheDir) + assert.Contains(t, fc.baseDir, "test-component") + + // Verify directory was created + _, err := os.Stat(customCacheDir) + assert.NoError(t, err, "Custom cache directory should be created") + }) + + t.Run("uses default UserCacheDir when env var not set", func(t *testing.T) { + os.Unsetenv("DATABRICKS_CACHE_DIR") + + cache := NewCache[string](ctx, "test-component", 60, nil) + fc, ok := cache.(*FileCache[string]) + require.True(t, ok) + + // Verify it's using the default path structure + userCacheDir, err := os.UserCacheDir() + require.NoError(t, err) + expectedPrefix := filepath.Join(userCacheDir, "databricks") + + assert.Contains(t, fc.baseDir, expectedPrefix) + }) + + t.Run("handles invalid cache dir path", func(t *testing.T) { + // Set an invalid path (no permissions) + t.Setenv("DATABRICKS_CACHE_DIR", "/root/invalid-cache-dir") + + cache := NewCache[string](ctx, "test-component", 60, nil) + _, ok := cache.(*NoopFileCache[string]) + require.True(t, ok) + }) +} + +func TestCacheIsolationByVersion(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + t.Setenv("DATABRICKS_CACHE_DIR", tempDir) + + // Create cache for component + cache := NewCache[string](ctx, "test-component", 60, nil) + fc, ok := cache.(*FileCache[string]) + require.True(t, ok) + + // Verify the cache path structure: // + // The path should contain the component name + assert.Contains(t, fc.baseDir, "test-component") + + // The path should be a subdirectory of tempDir + assert.Contains(t, fc.baseDir, tempDir) + + // Verify there's at least one intermediate directory between tempDir and component + // (the version directory) + relativePath, err := filepath.Rel(tempDir, fc.baseDir) + require.NoError(t, err) + + // Split by separator and count + pathParts := filepath.SplitList(relativePath) + // On most systems, SplitList is for PATH env var, not file paths + // Use strings.Split instead + if len(pathParts) == 1 { + pathParts = strings.Split(relativePath, string(filepath.Separator)) + } + + // Should have at least 2 parts: / + assert.GreaterOrEqual(t, len(pathParts), 2, "Cache path should include version directory: %s", relativePath) +} diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index 70141289ea0..eb695ca6729 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -5,10 +5,12 @@ import ( "os" "path/filepath" "runtime" + "strings" "sync/atomic" "testing" "time" + "github.com/databricks/cli/libs/env" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -17,11 +19,13 @@ func TestNewFileCache(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() cacheDir := filepath.Join(tempDir, "cache") + ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") + ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - cache, err := newFileCacheWithBaseDir[string](ctx, cacheDir, 60) - require.NoError(t, err) - assert.NotNil(t, cache) - assert.Equal(t, cacheDir, cache.baseDir) + cache := NewCache[string](ctx, "test-component", 60, nil) + fc, ok := cache.(*FileCache[string]) + require.True(t, ok) + assert.True(t, strings.HasPrefix(fc.baseDir, cacheDir)) // Verify directory was created info, err := os.Stat(cacheDir) @@ -51,32 +55,36 @@ func TestNewFileCacheWithExistingDirectory(t *testing.T) { err := os.MkdirAll(cacheDir, 0o700) require.NoError(t, err) - cache, err := newFileCacheWithBaseDir[string](ctx, cacheDir, 60) // 1 hour for tests + ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") + ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) + + cache := NewCache[string](ctx, "test-component", 60, nil) + fc, ok := cache.(*FileCache[string]) + require.True(t, ok) require.NoError(t, err) - assert.NotNil(t, cache) - assert.Equal(t, cacheDir, cache.baseDir) + assert.True(t, strings.HasPrefix(fc.baseDir, cacheDir)) } func TestNewFileCacheInvalidPath(t *testing.T) { ctx := context.Background() // Try to create cache in a location that should fail invalidPath := "/root/invalid/path/that/should/not/exist" + ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") + ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", invalidPath) - cache, err := newFileCacheWithBaseDir[string](ctx, invalidPath, 60) // 1 hour for tests - if err != nil { - assert.Nil(t, cache) - assert.Contains(t, err.Error(), "failed to create cache directory") - } + cache := NewCache[string](ctx, "test-component", 60, nil) + _, ok := cache.(*NoopFileCache[string]) + require.True(t, ok) } func TestFileCacheGetOrCompute(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](ctx, tempDir, 60) // 1 hour for tests - require.NoError(t, err) + cacheDir := filepath.Join(tempDir, "cache") + ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") + ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - // Enable cache for this test (default is measurement-only mode) - cache.cacheEnabled = true + cache := NewCache[string](ctx, "test-component", 60, nil) fingerprint := struct { Key string `json:"key"` @@ -112,8 +120,11 @@ func TestFileCacheGetOrCompute(t *testing.T) { func TestFileCacheGetOrComputeError(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](ctx, tempDir, 60) // 1 hour for tests - require.NoError(t, err) + cacheDir := filepath.Join(tempDir, "cache") + ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") + ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) + + cache := NewCache[string](ctx, "test-component", 60, nil) fingerprint := struct { Key string `json:"key"` @@ -134,8 +145,11 @@ func TestFileCacheGetOrComputeError(t *testing.T) { func TestFileCacheGetOrComputeConcurrency(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](ctx, tempDir, 60) // 1 hour for tests - require.NoError(t, err) + cacheDir := filepath.Join(tempDir, "cache") + ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") + ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) + + cache := NewCache[string](ctx, "test-component", 60, nil) fingerprint := struct { Key string `json:"key"` @@ -214,3 +228,189 @@ func TestFileCacheCleanupExpiredFiles(t *testing.T) { _, err = os.Stat(nonCacheFile) assert.False(t, os.IsNotExist(err), "Non-cache file should be ignored") } + +func TestFileCacheInvalidJSON(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + cache, err := newFileCacheWithBaseDir[string](ctx, tempDir, 60) + require.NoError(t, err) + + // Enable cache for this test + cache.cacheEnabled = true + + fingerprint := struct { + Key string `json:"key"` + }{ + Key: "test-invalid-json", + } + + // Manually write invalid JSON to the cache file + cacheKey, err := fingerprintToHash(fingerprint) + require.NoError(t, err) + cachePath := cache.getCachePath(cacheKey) + err = os.WriteFile(cachePath, []byte("invalid json {{{"), 0o600) + require.NoError(t, err) + + // GetOrCompute should fail open and recompute when cache contains invalid JSON + var computeCalls int32 + result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + atomic.AddInt32(&computeCalls, 1) + return "recomputed-value", nil + }) + + require.NoError(t, err) + assert.Equal(t, "recomputed-value", result) + assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls), "Should recompute when cache has invalid JSON") +} + +func TestFileCacheCorruptedData(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + cache, err := newFileCacheWithBaseDir[int](ctx, tempDir, 60) + require.NoError(t, err) + + // Enable cache for this test + cache.cacheEnabled = true + + fingerprint := struct { + Key string `json:"key"` + }{ + Key: "test-corrupted", + } + + // Write valid JSON but wrong type (string instead of int) + cacheKey, err := fingerprintToHash(fingerprint) + require.NoError(t, err) + cachePath := cache.getCachePath(cacheKey) + err = os.WriteFile(cachePath, []byte(`"not-an-integer"`), 0o600) + require.NoError(t, err) + + // GetOrCompute should fail open and recompute when cache type doesn't match + var computeCalls int32 + result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (int, error) { + atomic.AddInt32(&computeCalls, 1) + return 42, nil + }) + + require.NoError(t, err) + assert.Equal(t, 42, result) + assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls), "Should recompute when cache type is wrong") +} + +func TestFileCacheEmptyFingerprint(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + cache, err := newFileCacheWithBaseDir[string](ctx, tempDir, 60) + require.NoError(t, err) + + // Enable cache for this test + cache.cacheEnabled = true + + // Empty struct fingerprint is valid + fingerprint := struct{}{} + + var computeCalls int32 + result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + atomic.AddInt32(&computeCalls, 1) + return "value", nil + }) + require.NoError(t, err) + assert.Equal(t, "value", result) + + // Second call should use cache + result2, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + atomic.AddInt32(&computeCalls, 1) + return "should-not-be-called", nil + }) + require.NoError(t, err) + assert.Equal(t, "value", result2) + assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls), "Empty fingerprint should work with cache") +} + +func TestFileCacheMeasurementMode(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + cache, err := newFileCacheWithBaseDir[string](ctx, tempDir, 60) + require.NoError(t, err) + + // Keep cache disabled (measurement mode) + cache.cacheEnabled = false + + fingerprint := struct { + Key string `json:"key"` + }{ + Key: "test-measurement", + } + + // First call + var computeCalls int32 + result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + atomic.AddInt32(&computeCalls, 1) + return "computed-value", nil + }) + require.NoError(t, err) + assert.Equal(t, "computed-value", result) + assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) + + // Second call - in measurement mode, should always recompute + result2, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + atomic.AddInt32(&computeCalls, 1) + return "recomputed-value", nil + }) + require.NoError(t, err) + assert.Equal(t, "recomputed-value", result2) + assert.Equal(t, int32(2), atomic.LoadInt32(&computeCalls), "Measurement mode should always recompute") + + // But cache file should still exist + cacheFiles, err := filepath.Glob(filepath.Join(tempDir, "*.json")) + require.NoError(t, err) + assert.Len(t, cacheFiles, 1, "Cache file should be written even in measurement mode") +} + +func TestFileCacheReadPermissionError(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("Skipping permission test when running as root") + } + + ctx := context.Background() + tempDir := t.TempDir() + cache, err := newFileCacheWithBaseDir[string](ctx, tempDir, 60) + require.NoError(t, err) + + // Enable cache for this test + cache.cacheEnabled = true + + fingerprint := struct { + Key string `json:"key"` + }{ + Key: "test-permissions", + } + + // First, populate the cache + result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + return "cached-value", nil + }) + require.NoError(t, err) + assert.Equal(t, "cached-value", result) + + // Find the cache file and make it unreadable + cacheFiles, err := filepath.Glob(filepath.Join(tempDir, "*.json")) + require.NoError(t, err) + require.Len(t, cacheFiles, 1) + err = os.Chmod(cacheFiles[0], 0o000) + require.NoError(t, err) + + // Restore permissions after test + defer func() { _ = os.Chmod(cacheFiles[0], 0o600) }() + + // GetOrCompute should fail open and recompute when file is unreadable + var computeCalls int32 + result2, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + atomic.AddInt32(&computeCalls, 1) + return "recomputed-value", nil + }) + + require.NoError(t, err) + assert.Equal(t, "recomputed-value", result2) + assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls), "Should recompute when cache file is unreadable") +} diff --git a/libs/cache/fingerprint.go b/libs/cache/fingerprint.go new file mode 100644 index 00000000000..8f866405122 --- /dev/null +++ b/libs/cache/fingerprint.go @@ -0,0 +1,22 @@ +package cache + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" +) + +// fingerprintToHash converts any struct to a deterministic string representation for use as a cache key. +func fingerprintToHash(fingerprint any) (string, error) { + // Marshal map - json.Marshal sorts map keys alphabetically + data, err := json.Marshal(fingerprint) + if err != nil { + return "", fmt.Errorf("failed to marshal normalized fingerprint: %w", err) + } + + // Hash for consistent, reasonably-sized key. + // hash[:] converts the [32]byte array returned by Sum256 to a []byte slice. + hash := sha256.Sum256(data) + return hex.EncodeToString(hash[:]), nil +} diff --git a/libs/cache/fingerprint_test.go b/libs/cache/fingerprint_test.go new file mode 100644 index 00000000000..d2ba0c77885 --- /dev/null +++ b/libs/cache/fingerprint_test.go @@ -0,0 +1,33 @@ +package cache + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestFingerprintStability tests that the fingerprintToHash function returns the same hash for the same input. +func TestFingerprintStability(t *testing.T) { + fingerprint1 := struct { + Key string `json:"key"` + }{ + Key: "test-key", + } + + fingerprint2 := struct { + Key string `json:"key"` + }{ + Key: "test-key2", + } + + hash1, err := fingerprintToHash(fingerprint1) + require.NoError(t, err) + hash2, err := fingerprintToHash(fingerprint2) + require.NoError(t, err) + hash1ToCompare, err := fingerprintToHash(fingerprint1) + require.NoError(t, err) + + assert.Equal(t, hash1ToCompare, hash1) + assert.NotEqual(t, hash1, hash2) +} diff --git a/libs/cache/noop_file_cache.go b/libs/cache/noop_file_cache.go new file mode 100644 index 00000000000..9fe5cfe32b6 --- /dev/null +++ b/libs/cache/noop_file_cache.go @@ -0,0 +1,9 @@ +package cache + +import "context" + +type NoopFileCache[T any] struct{} + +func (c *NoopFileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compute func(ctx context.Context) (T, error)) (T, error) { + return compute(ctx) +} From 1c7a2dae7ff0e909ef11ca3f1702eac99f001fa2 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 1 Dec 2025 12:52:48 +0100 Subject: [PATCH 76/87] use separate cache test dir --- acceptance/acceptance_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 8dac7458bc9..0f905348d49 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -630,7 +630,7 @@ func runTest(t *testing.T, // Set unique cache folder for this test to avoid race conditions between parallel tests // Use test temp directory to avoid polluting user's cache - uniqueCacheDir := filepath.Join(tmpDir, ".cache") + uniqueCacheDir := filepath.Join(t.TempDir(), ".cache") cmd.Env = append(cmd.Env, "DATABRICKS_CACHE_DIR="+uniqueCacheDir) for _, key := range utils.SortedKeys(config.Env) { From 3e3777659f5d88491a016ad80132a813440e8b61 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 1 Dec 2025 13:08:16 +0100 Subject: [PATCH 77/87] fixes for windows --- libs/cache/file_cache.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index f4eb7bae20d..84f47dbab27 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -278,6 +278,15 @@ func (fc *FileCache[T]) writeToCache(ctx context.Context, cachePath string, data return } + if err := tempFile.Close(); err != nil { + log.Debugf(ctx, "[Local Cache] failed to close temp cache file: %v", err) + return + } + + // On Windows, os.Rename fails if target exists, so remove it first + // This is a best-effort operation - if it fails because file doesn't exist, that's fine + _ = os.Remove(cachePath) + // Atomically rename temp file to actual cache file if err := os.Rename(tempPath, cachePath); err != nil { log.Debugf(ctx, "[Local Cache] failed to rename temp cache file: %v", err) From 72d43b43a13f7bbf0271850e10db2d1f80aef3d6 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 1 Dec 2025 13:20:04 +0100 Subject: [PATCH 78/87] skip on windows --- libs/cache/file_cache_env_test.go | 5 +++++ libs/cache/file_cache_test.go | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/libs/cache/file_cache_env_test.go b/libs/cache/file_cache_env_test.go index 709746726f8..2da0419da45 100644 --- a/libs/cache/file_cache_env_test.go +++ b/libs/cache/file_cache_env_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "runtime" "strings" "sync/atomic" "testing" @@ -142,6 +143,10 @@ func TestCacheDirEnvVar(t *testing.T) { }) t.Run("handles invalid cache dir path", func(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping invalid path test on Windows - permission model differs") + } + // Set an invalid path (no permissions) t.Setenv("DATABRICKS_CACHE_DIR", "/root/invalid-cache-dir") diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index eb695ca6729..6f19abae2b9 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -66,6 +66,10 @@ func TestNewFileCacheWithExistingDirectory(t *testing.T) { } func TestNewFileCacheInvalidPath(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping invalid path test on Windows") + } + ctx := context.Background() // Try to create cache in a location that should fail invalidPath := "/root/invalid/path/that/should/not/exist" @@ -371,6 +375,9 @@ func TestFileCacheReadPermissionError(t *testing.T) { if os.Getuid() == 0 { t.Skip("Skipping permission test when running as root") } + if runtime.GOOS == "windows" { + t.Skip("Skipping permission test on Windows") + } ctx := context.Background() tempDir := t.TempDir() From 55f5438f58f1292a5075c4a8210cc15e484ff4fd Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 5 Dec 2025 12:30:43 +0100 Subject: [PATCH 79/87] fixes --- acceptance/cache/clear/output.txt | 3 --- acceptance/cache/simple/output.txt | 2 -- .../config/mutator/populate_current_user.go | 21 ++++--------------- libs/cache/file_cache_env_test.go | 14 ++++--------- 4 files changed, 8 insertions(+), 32 deletions(-) diff --git a/acceptance/cache/clear/output.txt b/acceptance/cache/clear/output.txt index 19ffef35021..bba37b6ccb0 100644 --- a/acceptance/cache/clear/output.txt +++ b/acceptance/cache/clear/output.txt @@ -1,13 +1,11 @@ === First call in a session is expected to be a cache miss: -[DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted) [DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing [DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result === Second call in a session is expected to be a cache hit -[DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit @@ -15,7 +13,6 @@ Cache cleared successfully from [TEST_TMP_DIR]/.cache === First call after a clear is expected to be a cache miss: -[DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted) [DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing diff --git a/acceptance/cache/simple/output.txt b/acceptance/cache/simple/output.txt index 40df5673b5f..093900b94b7 100644 --- a/acceptance/cache/simple/output.txt +++ b/acceptance/cache/simple/output.txt @@ -1,13 +1,11 @@ === First call in a session is expected to be a cache miss: -[DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted) [DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing [DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result === Second call in a session is expected to be a cache hit -[DEBUG_TIMESTAMP] Debug: [Local Cache] local cache is enabled [DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH] [DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index 74f13ea5381..bd17231281c 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -5,8 +5,6 @@ import ( "github.com/databricks/cli/libs/cache" - "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" @@ -46,25 +44,14 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. var err error fingerprint := b.GetUserFingerprint(ctx) - if !fingerprint.IsEmpty() { - log.Debugf(ctx, "[Local Cache] local cache is enabled") - me, err = m.cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (*iam.User, error) { - currentUser, err := w.CurrentUser.Me(ctx) - return currentUser, err - }) - } else { - log.Debugf(ctx, "[Local Cache] local cache is disabled") - me, err = w.CurrentUser.Me(ctx) - } - + me, err = m.cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (*iam.User, error) { + currentUser, err := w.CurrentUser.Me(ctx) + return currentUser, err + }) if err != nil { return diag.FromErr(err) } - if me == nil { - return diag.Errorf("could not find current user, but no error was returned") - } - b.Config.Workspace.CurrentUser = &config.User{ ShortName: iamutil.GetShortUserName(me), DomainFriendlyName: iamutil.GetShortUserDomainFriendlyName(me), diff --git a/libs/cache/file_cache_env_test.go b/libs/cache/file_cache_env_test.go index 2da0419da45..53244c02303 100644 --- a/libs/cache/file_cache_env_test.go +++ b/libs/cache/file_cache_env_test.go @@ -51,17 +51,11 @@ func TestCacheEnabledEnvVar(t *testing.T) { } for _, tt := range tests { + // Set up environment + if tt.envValue != "" { + t.Setenv("DATABRICKS_CACHE_ENABLED", tt.envValue) + } t.Run(tt.name, func(t *testing.T) { - // Set up environment - if tt.envValue != "" { - t.Setenv("DATABRICKS_CACHE_ENABLED", tt.envValue) - } else { - os.Unsetenv("DATABRICKS_CACHE_ENABLED") - } - - // Note: We can't reset sync.Once, so logging will only happen in the first test - // This is acceptable as we're testing behavior, not logging - // Create a unique subdirectory for this test testDir := filepath.Join(tempDir, tt.name) cache, err := newFileCacheWithBaseDir[string](ctx, testDir, 60) From 82391996f87ae9baf35e5d3235b6ad5beda51342 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 5 Dec 2025 15:52:46 +0100 Subject: [PATCH 80/87] avoid double computations --- libs/cache/file_cache.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 84f47dbab27..7bff0c09e1a 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -174,6 +174,10 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu cachePath := fc.getCachePath(cacheKey) + // Acquire lock to prevent concurrent and double computations and writes for the same cache key + fc.mu.Lock() + defer fc.mu.Unlock() + // Try to read from disk cache cachedData, cacheExists := fc.readFromCache(ctx, cachePath) @@ -191,10 +195,6 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu fc.addTelemetryMetric("local.cache.miss") } - // Acquire lock to prevent concurrent computations and writes for the same cache key - fc.mu.Lock() - defer fc.mu.Unlock() - // Compute the value and measure timing start := time.Now() result, err := compute(ctx) From ae498cf79dc08f148d94008ba2939d5b218033db Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 8 Dec 2025 14:17:44 +0100 Subject: [PATCH 81/87] remove repls --- .../resource_deps/job_tasks/out.telemetry.direct.txt | 4 ++-- .../resource_deps/job_tasks/out.telemetry.terraform.txt | 4 ++-- acceptance/bundle/resource_deps/resources_var/output.txt | 4 ++-- acceptance/bundle/resource_deps/test.toml | 4 ---- .../bundle/telemetry/deploy-compute-type/output.txt | 8 ++++---- .../bundle/telemetry/deploy-experimental/output.txt | 4 ++-- .../bundle/telemetry/deploy-name-prefix/custom/output.txt | 4 ++-- .../deploy-name-prefix/mode-development/output.txt | 4 ++-- .../bundle/telemetry/deploy-whl-artifacts/output.txt | 8 ++++---- acceptance/bundle/telemetry/deploy/out.telemetry.txt | 4 ++-- acceptance/bundle/telemetry/test.toml | 4 ---- 11 files changed, 22 insertions(+), 30 deletions(-) diff --git a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt index 9c10edcc8b7..410528fdf3f 100644 --- a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt +++ b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt @@ -2,8 +2,8 @@ experimental.use_legacy_run_as false has_classic_interactive_compute false has_classic_job_compute false has_serverless_compute true -local.cache.(redacted) -local.cache.(redacted) +local.cache.attempt true +local.cache.miss true presets_name_prefix_is_set false python_wheel_wrapper_is_set false resref_jobs.tags.* true diff --git a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt index ed3c27c9540..50371a06442 100644 --- a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt +++ b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt @@ -2,8 +2,8 @@ experimental.use_legacy_run_as false has_classic_interactive_compute false has_classic_job_compute false has_serverless_compute true -local.cache.(redacted) -local.cache.(redacted) +local.cache.attempt true +local.cache.miss true presets_name_prefix_is_set false python_wheel_wrapper_is_set false resref_jobs.tags.* true diff --git a/acceptance/bundle/resource_deps/resources_var/output.txt b/acceptance/bundle/resource_deps/resources_var/output.txt index 51bdc0789c9..cd34790c2ca 100644 --- a/acceptance/bundle/resource_deps/resources_var/output.txt +++ b/acceptance/bundle/resource_deps/resources_var/output.txt @@ -40,8 +40,8 @@ experimental.use_legacy_run_as false has_classic_interactive_compute false has_classic_job_compute false has_serverless_compute false -local.cache.(redacted) -local.cache.(redacted) +local.cache.attempt true +local.cache.hit true presets_name_prefix_is_set true python_wheel_wrapper_is_set false resref_volumes.catalog_name true diff --git a/acceptance/bundle/resource_deps/test.toml b/acceptance/bundle/resource_deps/test.toml index 78ef97636e9..a2b2d9fc33f 100644 --- a/acceptance/bundle/resource_deps/test.toml +++ b/acceptance/bundle/resource_deps/test.toml @@ -5,7 +5,3 @@ Ignore = [ ".databricks", ".gitignore", ] - -[[Repls]] -Old = 'local\.cache\.(.*)' -New = 'local.cache.(redacted)' diff --git a/acceptance/bundle/telemetry/deploy-compute-type/output.txt b/acceptance/bundle/telemetry/deploy-compute-type/output.txt index 057c39be997..a424df8ce1d 100644 --- a/acceptance/bundle/telemetry/deploy-compute-type/output.txt +++ b/acceptance/bundle/telemetry/deploy-compute-type/output.txt @@ -14,11 +14,11 @@ Deployment complete! >>> cat out.requests.txt [ { - "key": "local.cache.(redacted) + "key": "local.cache.attempt", "value": true }, { - "key": "local.cache.(redacted) + "key": "local.cache.miss", "value": true }, { @@ -56,11 +56,11 @@ Deployment complete! ] [ { - "key": "local.cache.(redacted) + "key": "local.cache.attempt", "value": true }, { - "key": "local.cache.(redacted) + "key": "local.cache.hit", "value": true }, { diff --git a/acceptance/bundle/telemetry/deploy-experimental/output.txt b/acceptance/bundle/telemetry/deploy-experimental/output.txt index 4bc1fa21121..05bc64e441d 100644 --- a/acceptance/bundle/telemetry/deploy-experimental/output.txt +++ b/acceptance/bundle/telemetry/deploy-experimental/output.txt @@ -13,11 +13,11 @@ Deployment complete! { "bool_values": [ { - "key": "local.cache.(redacted) + "key": "local.cache.attempt", "value": true }, { - "key": "local.cache.(redacted) + "key": "local.cache.miss", "value": true }, { diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt b/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt index e9c44168984..31ff8e9cf7e 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt @@ -9,11 +9,11 @@ Deployment complete! { "bool_values": [ { - "key": "local.cache.(redacted) + "key": "local.cache.attempt", "value": true }, { - "key": "local.cache.(redacted) + "key": "local.cache.miss", "value": true }, { diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt index 55cbc6c1f04..39b671bec32 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt @@ -9,11 +9,11 @@ Deployment complete! { "bool_values": [ { - "key": "local.cache.(redacted) + "key": "local.cache.attempt", "value": true }, { - "key": "local.cache.(redacted) + "key": "local.cache.miss", "value": true }, { diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt b/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt index aa5a8477e63..a9b8ce4ae6e 100644 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt @@ -17,11 +17,11 @@ Deployment complete! { "bool_values": [ { - "key": "local.cache.(redacted) + "key": "local.cache.attempt", "value": true }, { - "key": "local.cache.(redacted) + "key": "local.cache.miss", "value": true }, { @@ -57,11 +57,11 @@ Deployment complete! { "bool_values": [ { - "key": "local.cache.(redacted) + "key": "local.cache.attempt", "value": true }, { - "key": "local.cache.(redacted) + "key": "local.cache.hit", "value": true }, { diff --git a/acceptance/bundle/telemetry/deploy/out.telemetry.txt b/acceptance/bundle/telemetry/deploy/out.telemetry.txt index b932d24e27f..f945233dd16 100644 --- a/acceptance/bundle/telemetry/deploy/out.telemetry.txt +++ b/acceptance/bundle/telemetry/deploy/out.telemetry.txt @@ -43,11 +43,11 @@ "target_count": 1, "bool_values": [ { - "key": "local.cache.(redacted) + "key": "local.cache.attempt", "value": true }, { - "key": "local.cache.(redacted) + "key": "local.cache.miss", "value": true }, { diff --git a/acceptance/bundle/telemetry/test.toml b/acceptance/bundle/telemetry/test.toml index 4c525096880..d47cfd33e3d 100644 --- a/acceptance/bundle/telemetry/test.toml +++ b/acceptance/bundle/telemetry/test.toml @@ -12,10 +12,6 @@ New = '"execution_time_ms": SMALL_INT,' Old = '(linux|darwin|windows)' New = '[OS]' -[[Repls]] -Old = 'local\.cache\.(.*)' -New = 'local.cache.(redacted)' - [[Repls]] Old = '"local_cache_measurements_ms": \[[^\]]*\]' New = '"local_cache_measurements_ms": [...redacted...]' From 6ec52193577bfd6f8f11e01af13fbecbc486a64f Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 9 Dec 2025 11:34:54 +0100 Subject: [PATCH 82/87] hardcode hash and refactor --- internal/build/info.go | 13 +++++++++++++ libs/cache/file_cache.go | 15 +-------------- libs/cache/fingerprint_test.go | 1 + 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/internal/build/info.go b/internal/build/info.go index 8ddf4d4f2e6..3f03c6f6a7c 100644 --- a/internal/build/info.go +++ b/internal/build/info.go @@ -4,6 +4,7 @@ import ( "fmt" "runtime/debug" "strconv" + "strings" "sync" "time" @@ -29,6 +30,18 @@ type Info struct { BuildTime time.Time } +// sanitizeVersion removes characters from version string that might be problematic in file paths. +// Particularly important for Windows which has restrictions on certain characters. +func (i Info) GetSanitizedVersion() string { + // Replace + with - (used in version metadata like "1.0.0+abc123") + version := strings.ReplaceAll(i.Version, "+", "-") + // Remove any other potentially problematic characters + version = strings.ReplaceAll(version, ":", "-") + version = strings.ReplaceAll(version, "/", "-") + version = strings.ReplaceAll(version, "\\", "-") + return version +} + var info Info var once sync.Once diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 7bff0c09e1a..917f9c2efe0 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -6,7 +6,6 @@ import ( "fmt" "os" "path/filepath" - "strings" "sync" "time" @@ -109,18 +108,6 @@ func getCacheBaseDir(ctx context.Context) (string, error) { return filepath.Join(userCacheDir, "databricks"), nil } -// sanitizeVersion removes characters from version string that might be problematic in file paths. -// Particularly important for Windows which has restrictions on certain characters. -func sanitizeVersion(version string) string { - // Replace + with - (used in version metadata like "1.0.0+abc123") - version = strings.ReplaceAll(version, "+", "-") - // Remove any other potentially problematic characters - version = strings.ReplaceAll(version, ":", "-") - version = strings.ReplaceAll(version, "/", "-") - version = strings.ReplaceAll(version, "\\", "-") - return version -} - // NewCache creates a new file-based cache using UserCacheDir() + "databricks" + version + cached component name. // Including the CLI version in the path ensures cache isolation across different CLI versions. // By default, the cache operates in measurement-only mode (cacheEnabled=false), which means it will: @@ -137,7 +124,7 @@ func NewCache[T any](ctx context.Context, component string, expiryMinutes int, m // Include CLI version in cache path to avoid issues across versions // Sanitize version string for use in file paths - version := sanitizeVersion(build.GetInfo().Version) + version := build.GetInfo().GetSanitizedVersion() baseDir := filepath.Join(cacheBaseDir, version, component) fc, err := newFileCacheWithBaseDir[T](ctx, baseDir, expiryMinutes) if err != nil { diff --git a/libs/cache/fingerprint_test.go b/libs/cache/fingerprint_test.go index d2ba0c77885..ae64564cc66 100644 --- a/libs/cache/fingerprint_test.go +++ b/libs/cache/fingerprint_test.go @@ -23,6 +23,7 @@ func TestFingerprintStability(t *testing.T) { hash1, err := fingerprintToHash(fingerprint1) require.NoError(t, err) + require.Equal(t, hash1, "1b329dc07a9fa87da7480f6b10cc917a40a4f460ac82aea3d09df477764f3101") hash2, err := fingerprintToHash(fingerprint2) require.NoError(t, err) hash1ToCompare, err := fingerprintToHash(fingerprint1) From 16244192e17a8dc2213480e888390cacb6b0d176 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 9 Dec 2025 12:09:20 +0100 Subject: [PATCH 83/87] fix lint --- libs/cache/fingerprint_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/cache/fingerprint_test.go b/libs/cache/fingerprint_test.go index ae64564cc66..8a85f7944ce 100644 --- a/libs/cache/fingerprint_test.go +++ b/libs/cache/fingerprint_test.go @@ -23,7 +23,7 @@ func TestFingerprintStability(t *testing.T) { hash1, err := fingerprintToHash(fingerprint1) require.NoError(t, err) - require.Equal(t, hash1, "1b329dc07a9fa87da7480f6b10cc917a40a4f460ac82aea3d09df477764f3101") + require.Equal(t, "1b329dc07a9fa87da7480f6b10cc917a40a4f460ac82aea3d09df477764f3101", hash1) hash2, err := fingerprintToHash(fingerprint2) require.NoError(t, err) hash1ToCompare, err := fingerprintToHash(fingerprint1) From 512433c3f7054629d01a0128e7495bb27bd4879d Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 10 Dec 2025 09:57:42 +0100 Subject: [PATCH 84/87] refactor to use generic function --- bundle/bundle.go | 5 ++ bundle/config/mutator/initialize_cache.go | 26 ++++++ .../config/mutator/initialize_cache_test.go | 26 ++++++ .../config/mutator/populate_current_user.go | 17 +--- bundle/phases/initialize.go | 5 ++ libs/cache/cache.go | 64 ++++++++++++--- libs/cache/file_cache.go | 80 ++++++++----------- libs/cache/file_cache_env_test.go | 26 +++--- libs/cache/file_cache_expiry_test.go | 22 ++--- libs/cache/file_cache_test.go | 78 ++++++++++-------- libs/cache/noop_file_cache.go | 7 +- 11 files changed, 225 insertions(+), 131 deletions(-) create mode 100644 bundle/config/mutator/initialize_cache.go create mode 100644 bundle/config/mutator/initialize_cache_test.go diff --git a/bundle/bundle.go b/bundle/bundle.go index 2732f085650..5102e739f40 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -20,6 +20,7 @@ import ( "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/bundle/metadata" "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/cache" "github.com/databricks/cli/libs/fileset" "github.com/databricks/cli/libs/locker" "github.com/databricks/cli/libs/log" @@ -157,6 +158,10 @@ type Bundle struct { // The implementation depends on the cloud being targeted. Tagging tags.Cloud + // Cache is used for caching API responses (e.g., current user). + // By default, operates in measurement-only mode. Set DATABRICKS_CACHE_ENABLED=true to enable actual caching. + Cache *cache.Cache + Metrics Metrics } diff --git a/bundle/config/mutator/initialize_cache.go b/bundle/config/mutator/initialize_cache.go new file mode 100644 index 00000000000..b55ea5e29f8 --- /dev/null +++ b/bundle/config/mutator/initialize_cache.go @@ -0,0 +1,26 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/cache" + "github.com/databricks/cli/libs/diag" +) + +type initializeCache struct{} + +// InitializeCache initializes the bundle cache which can be used to cache API responses. +func InitializeCache() bundle.Mutator { + return &initializeCache{} +} + +func (m *initializeCache) Name() string { + return "InitializeCache" +} + +func (m *initializeCache) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + // Initialize cache with 30 minute expiry for user information + b.Cache = cache.NewCache(ctx, "user", 30, &b.Metrics) + return nil +} diff --git a/bundle/config/mutator/initialize_cache_test.go b/bundle/config/mutator/initialize_cache_test.go new file mode 100644 index 00000000000..f6381fd906c --- /dev/null +++ b/bundle/config/mutator/initialize_cache_test.go @@ -0,0 +1,26 @@ +package mutator_test + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInitializeCache(t *testing.T) { + ctx := context.Background() + b := &bundle.Bundle{} + + // Cache should be nil initially + assert.Nil(t, b.Cache) + + // Apply the mutator + diags := bundle.Apply(ctx, b, mutator.InitializeCache()) + require.NoError(t, diags.Error()) + + // Cache should now be initialized + assert.NotNil(t, b.Cache) +} diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index bd17231281c..ef654d87507 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -3,32 +3,22 @@ package mutator import ( "context" - "github.com/databricks/cli/libs/cache" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/cache" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/iamutil" "github.com/databricks/cli/libs/tags" "github.com/databricks/databricks-sdk-go/service/iam" ) -type populateCurrentUser struct { - cache cache.Cache[*iam.User] -} +type populateCurrentUser struct{} // PopulateCurrentUser sets the `current_user` property on the workspace. func PopulateCurrentUser() bundle.Mutator { return &populateCurrentUser{} } -// initializeCache sets up the cache for authorization headers if not already initialized. -// By default, cache operates in measurement-only mode to gather metrics about potential savings. -// Set DATABRICKS_CACHE_ENABLED=true to enable actual caching. -func (m *populateCurrentUser) initializeCache(ctx context.Context, b *bundle.Bundle) { - m.cache = cache.NewCache[*iam.User](ctx, "auth", 30, &b.Metrics) -} - func (m *populateCurrentUser) Name() string { return "PopulateCurrentUser" } @@ -37,14 +27,13 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. if b.Config.Workspace.CurrentUser != nil { return nil } - m.initializeCache(ctx, b) w := b.WorkspaceClient() var me *iam.User var err error fingerprint := b.GetUserFingerprint(ctx) - me, err = m.cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (*iam.User, error) { + me, err = cache.GetOrCompute(b.Cache, ctx, fingerprint, func(ctx context.Context) (*iam.User, error) { currentUser, err := w.CurrentUser.Me(ctx) return currentUser, err }) diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index cfacba015a7..761714b48e6 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -50,7 +50,12 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Updates (typed) b.Config.{Sync,Include,Exclude} they set to be relative to SyncRootPath instead of bundle root mutator.SyncInferRoot(), + // Updates (typed): b.Cache (initializes cache for API responses) + // Initialize cache before any mutator that might need it + mutator.InitializeCache(), + // Reads (typed): b.Config.Workspace.CurrentUser (checks if it's already set) + // Reads (typed): b.Cache (uses cache for current user API call) // Updates (typed): b.Config.Workspace.CurrentUser (sets user information from API) // Updates (typed): b.Tagging (configures tagging object based on current cloud) mutator.PopulateCurrentUser(), diff --git a/libs/cache/cache.go b/libs/cache/cache.go index 5eec234666a..42c6672b7fe 100644 --- a/libs/cache/cache.go +++ b/libs/cache/cache.go @@ -2,18 +2,60 @@ package cache import ( "context" + "encoding/json" + "fmt" ) -// Cache provides an abstract interface for caching content to local disk. -// Implementations should handle storing and retrieving cached components -// using fingerprints for cache invalidation. +// cacheImpl is the internal interface for cache implementations. +type cacheImpl interface { + getOrComputeJSON(ctx context.Context, fingerprint any, compute func(ctx context.Context) ([]byte, error)) ([]byte, bool, error) +} + +// Cache provides a concrete cache that works with any type through the generic GetOrCompute function. +// Create with NewCache() and use GetOrCompute[T]() for type-safe caching. +type Cache struct { + impl cacheImpl +} + +// GetOrCompute retrieves cached content for the given fingerprint, or computes it using the provided function. +// If the content is found in cache, it is returned directly. +// If not found, the compute function is called, its result is cached, and then returned. +// The fingerprint can be any struct that will be serialized deterministically for cache key generation. // Cache operations fail open: if caching fails, the compute function is still called. -type Cache[T any] interface { - // GetOrCompute retrieves cached content for the given fingerprint, or computes it using the provided function. - // If the content is found in cache, it is returned directly. - // If not found, the compute function is called, its result is cached, and then returned. - // The fingerprint can be any struct that will be serialized deterministically for cache key generation. - // Cache failures do not block computation - if caching fails, compute is called anyway. - // Returns an error only if the compute function fails. - GetOrCompute(ctx context.Context, fingerprint any, compute func(ctx context.Context) (T, error)) (T, error) +// Returns an error only if the compute function fails. +// The type parameter T is inferred from the compute function's return type. +func GetOrCompute[T any](c *Cache, ctx context.Context, fingerprint any, compute func(ctx context.Context) (T, error)) (T, error) { + var zero T + + // Wrap the compute function to serialize to JSON + computeJSON := func(ctx context.Context) ([]byte, error) { + result, err := compute(ctx) + if err != nil { + return nil, err + } + return json.Marshal(result) + } + + // Call the internal method + jsonBytes, fromCache, err := c.impl.getOrComputeJSON(ctx, fingerprint, computeJSON) + if err != nil { + return zero, err + } + + // Unmarshal into the target type + var result T + if err := json.Unmarshal(jsonBytes, &result); err != nil { + // If we got corrupted data from cache, fail open and recompute + if fromCache { + result, computeErr := compute(ctx) + if computeErr != nil { + return zero, computeErr + } + return result, nil + } + // If compute function returned invalid JSON, that's a real error + return zero, fmt.Errorf("failed to unmarshal computed data: %w", err) + } + + return result, nil } diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 917f9c2efe0..8b2ceb8439f 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -2,7 +2,6 @@ package cache import ( "context" - "encoding/json" "fmt" "os" "path/filepath" @@ -20,8 +19,8 @@ type Metrics interface { SetDurationValue(key string, value time.Duration) } -// FileCache implements the Cache interface using local disk storage. -type FileCache[T any] struct { +// fileCache implements the cacheImpl interface using local disk storage. +type fileCache struct { baseDir string expiryMinutes int mu sync.Mutex @@ -30,12 +29,12 @@ type FileCache[T any] struct { } // newFileCacheWithBaseDir creates a new file-based cache that stores data in the specified directory. -func newFileCacheWithBaseDir[T any](ctx context.Context, baseDir string, expiryMinutes int) (*FileCache[T], error) { +func newFileCacheWithBaseDir(ctx context.Context, baseDir string, expiryMinutes int) (*fileCache, error) { if err := os.MkdirAll(baseDir, 0o700); err != nil { return nil, fmt.Errorf("failed to create cache directory: %w", err) } - fc := &FileCache[T]{ + fc := &fileCache{ baseDir: baseDir, expiryMinutes: expiryMinutes, } @@ -47,7 +46,7 @@ func newFileCacheWithBaseDir[T any](ctx context.Context, baseDir string, expiryM } // isExpired checks if a file with the given modification time has expired. -func (fc *FileCache[T]) isExpired(modTime time.Time) bool { +func (fc *fileCache) isExpired(modTime time.Time) bool { expiryThreshold := time.Now().Add(-time.Duration(fc.expiryMinutes) * time.Minute) return modTime.Before(expiryThreshold) } @@ -55,7 +54,7 @@ func (fc *FileCache[T]) isExpired(modTime time.Time) bool { // cleanupExpiredFiles removes expired cache files from disk based on file modification time. // This runs synchronously once when the cache is created. // Files older than expiryMinutes are deleted. -func (fc *FileCache[T]) cleanupExpiredFiles(ctx context.Context) { +func (fc *fileCache) cleanupExpiredFiles(ctx context.Context) { err := filepath.Walk(fc.baseDir, func(path string, info os.FileInfo, err error) error { if err != nil { // Log walk errors but continue cleanup @@ -116,44 +115,46 @@ func getCacheBaseDir(ctx context.Context) (string, error) { // - Emit metrics about potential savings // - Always compute the value (never actually use the cache) // Set DATABRICKS_CACHE_ENABLED=true to enable actual caching. -func NewCache[T any](ctx context.Context, component string, expiryMinutes int, metrics Metrics) Cache[T] { +// The returned cache can handle multiple types through the generic GetOrCompute function. +func NewCache(ctx context.Context, component string, expiryMinutes int, metrics Metrics) *Cache { cacheBaseDir, err := getCacheBaseDir(ctx) if err != nil { - return &NoopFileCache[T]{} + return &Cache{impl: &noopFileCache{}} } // Include CLI version in cache path to avoid issues across versions // Sanitize version string for use in file paths version := build.GetInfo().GetSanitizedVersion() baseDir := filepath.Join(cacheBaseDir, version, component) - fc, err := newFileCacheWithBaseDir[T](ctx, baseDir, expiryMinutes) + fc, err := newFileCacheWithBaseDir(ctx, baseDir, expiryMinutes) if err != nil { - return &NoopFileCache[T]{} + return &Cache{impl: &noopFileCache{}} } fc.metrics = metrics // Check if cache is enabled; default is false (measurement-only mode) fc.cacheEnabled = env.Get(ctx, "DATABRICKS_CACHE_ENABLED") == "true" - return fc + return &Cache{impl: fc} } -func (fc *FileCache[T]) addTelemetryMetric(key string) { +func (fc *fileCache) addTelemetryMetric(key string) { if fc.metrics != nil { fc.metrics.SetBoolValue(key, true) } } -// GetOrCompute retrieves cached content or computes it using the provided function. +// getOrComputeJSON retrieves cached content or computes it using the provided function. // Cache operations fail open: if caching fails, the compute function is still called. // When cacheEnabled is false, the cache checks if values exist and measures potential time savings, // but always computes and never returns cached values. -func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compute func(ctx context.Context) (T, error)) (T, error) { +func (fc *fileCache) getOrComputeJSON(ctx context.Context, fingerprint any, compute func(ctx context.Context) ([]byte, error)) ([]byte, bool, error) { // Convert fingerprint to deterministic hash - this is our cache key cacheKey, err := fingerprintToHash(fingerprint) if err != nil { // Fail open: if we can't generate cache key, just compute directly log.Debugf(ctx, "[Local Cache] failed to generate cache key, computing without cache: %v", err) - return compute(ctx) + result, err := compute(ctx) + return result, false, err } log.Debugf(ctx, "[Local Cache] using cache key: %s", cacheKey) @@ -166,7 +167,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu defer fc.mu.Unlock() // Try to read from disk cache - cachedData, cacheExists := fc.readFromCache(ctx, cachePath) + cachedData, cacheExists := fc.readFromCacheJSON(ctx, cachePath) // Record metrics if cacheExists { @@ -175,7 +176,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu // If cache is enabled, return the cached value if fc.cacheEnabled { - return cachedData, nil + return cachedData, true, nil } } else { log.Debugf(ctx, "[Local Cache] cache miss, computing") @@ -188,7 +189,7 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu if err != nil { log.Debugf(ctx, "[Local Cache] error while computing: %v", err) fc.addTelemetryMetric("local.cache.error") - return result, err + return result, false, err } // Record duration metrics @@ -200,53 +201,38 @@ func (fc *FileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compu log.Debugf(ctx, "[Local Cache] computed and stored result") // Write to disk cache (failures are silent - cache write errors don't affect the result) - fc.writeToCache(ctx, cachePath, result) + fc.writeToCacheJSON(ctx, cachePath, result) - return result, nil + return result, false, nil } -// readFromCache attempts to read and deserialize data from the cache file. +// readFromCacheJSON attempts to read data from the cache file. // Expiry is checked using file modification time for consistency with cleanup. -func (fc *FileCache[T]) readFromCache(ctx context.Context, cachePath string) (T, bool) { - var zero T - +func (fc *fileCache) readFromCacheJSON(ctx context.Context, cachePath string) ([]byte, bool) { // Check file modification time for expiry info, err := os.Stat(cachePath) if err != nil { log.Debugf(ctx, "[Local Cache] failed to stat cache file: %v", err) - return zero, false + return nil, false } if fc.isExpired(info.ModTime()) { - return zero, false + return nil, false } - // Read and deserialize the data + // Read the data data, err := os.ReadFile(cachePath) if err != nil { log.Debugf(ctx, "[Local Cache] failed to read cache file: %v", err) - return zero, false - } - - var result T - if err := json.Unmarshal(data, &result); err != nil { - log.Debugf(ctx, "[Local Cache] failed to deserialize data: %v", err) - return zero, false + return nil, false } - return result, true + return data, true } -// writeToCache serializes and writes data to the cache file atomically. +// writeToCacheJSON writes data to the cache file atomically. // Uses atomic write: writes to temp file first, then renames to actual cache file. -func (fc *FileCache[T]) writeToCache(ctx context.Context, cachePath string, data any) { - // Serialize the data directly - serializedData, err := json.Marshal(data) - if err != nil { - log.Debugf(ctx, "[Local Cache] failed to serialize data: %v", err) - return // Silently fail on serialization errors - } - +func (fc *fileCache) writeToCacheJSON(ctx context.Context, cachePath string, data []byte) { // Create temporary file in the same directory for atomic operation tempFile, err := os.CreateTemp(fc.baseDir, ".cache-*.tmp") if err != nil { @@ -260,7 +246,7 @@ func (fc *FileCache[T]) writeToCache(ctx context.Context, cachePath string, data }() // Write data to temp file - if _, err := tempFile.Write(serializedData); err != nil { + if _, err := tempFile.Write(data); err != nil { log.Debugf(ctx, "[Local Cache] failed to write to temp cache file: %v", err) return } @@ -281,6 +267,6 @@ func (fc *FileCache[T]) writeToCache(ctx context.Context, cachePath string, data } // getCachePath returns the full path to the cache file for a given cache key. -func (fc *FileCache[T]) getCachePath(cacheKey string) string { +func (fc *fileCache) getCachePath(cacheKey string) string { return filepath.Join(fc.baseDir, cacheKey+".json") } diff --git a/libs/cache/file_cache_env_test.go b/libs/cache/file_cache_env_test.go index 53244c02303..d1b205f2362 100644 --- a/libs/cache/file_cache_env_test.go +++ b/libs/cache/file_cache_env_test.go @@ -58,11 +58,13 @@ func TestCacheEnabledEnvVar(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // Create a unique subdirectory for this test testDir := filepath.Join(tempDir, tt.name) - cache, err := newFileCacheWithBaseDir[string](ctx, testDir, 60) + fc, err := newFileCacheWithBaseDir(ctx, testDir, 60) require.NoError(t, err) // Set cacheEnabled based on env var (simulate NewFileCache behavior) - cache.cacheEnabled = env.Get(ctx, "DATABRICKS_CACHE_ENABLED") == "true" + fc.cacheEnabled = env.Get(ctx, "DATABRICKS_CACHE_ENABLED") == "true" + + cache := &Cache{impl: fc} fingerprint := struct { Key string `json:"key"` @@ -72,7 +74,7 @@ func TestCacheEnabledEnvVar(t *testing.T) { // First call - should always compute var computeCalls int32 - result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + result, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "computed-value", nil }) @@ -81,7 +83,7 @@ func TestCacheEnabledEnvVar(t *testing.T) { assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) // Second call - should use cache only if enabled - result2, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + result2, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "should-not-be-called", nil }) @@ -108,8 +110,8 @@ func TestCacheDirEnvVar(t *testing.T) { customCacheDir := filepath.Join(tempDir, "custom-cache") t.Setenv("DATABRICKS_CACHE_DIR", customCacheDir) - cache := NewCache[string](ctx, "test-component", 60, nil) - fc, ok := cache.(*FileCache[string]) + cache := NewCache(ctx, "test-component", 60, nil) + fc, ok := cache.impl.(*fileCache) require.True(t, ok) // Verify the cache directory is under the custom path @@ -124,8 +126,8 @@ func TestCacheDirEnvVar(t *testing.T) { t.Run("uses default UserCacheDir when env var not set", func(t *testing.T) { os.Unsetenv("DATABRICKS_CACHE_DIR") - cache := NewCache[string](ctx, "test-component", 60, nil) - fc, ok := cache.(*FileCache[string]) + cache := NewCache(ctx, "test-component", 60, nil) + fc, ok := cache.impl.(*fileCache) require.True(t, ok) // Verify it's using the default path structure @@ -144,8 +146,8 @@ func TestCacheDirEnvVar(t *testing.T) { // Set an invalid path (no permissions) t.Setenv("DATABRICKS_CACHE_DIR", "/root/invalid-cache-dir") - cache := NewCache[string](ctx, "test-component", 60, nil) - _, ok := cache.(*NoopFileCache[string]) + cache := NewCache(ctx, "test-component", 60, nil) + _, ok := cache.impl.(*noopFileCache) require.True(t, ok) }) } @@ -156,8 +158,8 @@ func TestCacheIsolationByVersion(t *testing.T) { t.Setenv("DATABRICKS_CACHE_DIR", tempDir) // Create cache for component - cache := NewCache[string](ctx, "test-component", 60, nil) - fc, ok := cache.(*FileCache[string]) + cache := NewCache(ctx, "test-component", 60, nil) + fc, ok := cache.impl.(*fileCache) require.True(t, ok) // Verify the cache path structure: // diff --git a/libs/cache/file_cache_expiry_test.go b/libs/cache/file_cache_expiry_test.go index 02da219f177..d21f09946da 100644 --- a/libs/cache/file_cache_expiry_test.go +++ b/libs/cache/file_cache_expiry_test.go @@ -17,11 +17,13 @@ func TestFileCacheExpiryBehavior(t *testing.T) { tempDir := t.TempDir() // Create cache with 1 minute expiry - cache, err := newFileCacheWithBaseDir[string](ctx, tempDir, 1) + fc, err := newFileCacheWithBaseDir(ctx, tempDir, 1) require.NoError(t, err) // Enable cache for this test (default is measurement-only mode) - cache.cacheEnabled = true + fc.cacheEnabled = true + + cache := &Cache{impl: fc} fingerprint := struct { Key string `json:"key"` @@ -30,7 +32,7 @@ func TestFileCacheExpiryBehavior(t *testing.T) { } // Compute and store a value - result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + result, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { return "test-value", nil }) require.NoError(t, err) @@ -58,7 +60,7 @@ func TestFileCacheExpiryBehavior(t *testing.T) { // Verify GetOrCompute treats it as a cache miss and recomputes callCount := 0 - result, err = cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + result, err = GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { callCount++ return "recomputed-value", nil }) @@ -67,11 +69,11 @@ func TestFileCacheExpiryBehavior(t *testing.T) { assert.Equal(t, 1, callCount, "Should have called compute function once due to cache expiry") } -// TestReadFromCacheRespectsExpiry tests that readFromCache returns false for expired entries based on mtime +// TestReadFromCacheRespectsExpiry tests that readFromCacheJSON returns false for expired entries based on mtime func TestReadFromCacheRespectsExpiry(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](ctx, tempDir, 1) // 1 minute expiry + cache, err := newFileCacheWithBaseDir(ctx, tempDir, 1) // 1 minute expiry require.NoError(t, err) // Create an expired cache file by setting its mtime to 2 hours ago @@ -81,16 +83,16 @@ func TestReadFromCacheRespectsExpiry(t *testing.T) { require.NoError(t, os.Chtimes(expiredFile, oldTime, oldTime)) // Try to read from expired cache - should return false - result, found := cache.readFromCache(ctx, expiredFile) + result, found := cache.readFromCacheJSON(ctx, expiredFile) assert.False(t, found, "Should not find expired cache entry") - assert.Equal(t, "", result, "Result should be zero value for expired entry") + assert.Nil(t, result, "Result should be nil for expired entry") // Create a valid (non-expired) cache file with recent mtime validFile := filepath.Join(tempDir, "valid.json") require.NoError(t, os.WriteFile(validFile, []byte(`"valid-value"`), 0o644)) // Try to read from valid cache - should return true - result, found = cache.readFromCache(ctx, validFile) + result, found = cache.readFromCacheJSON(ctx, validFile) assert.True(t, found, "Should find valid cache entry") - assert.Equal(t, "valid-value", result, "Should return correct value for valid entry") + assert.Equal(t, `"valid-value"`, string(result), "Should return correct value for valid entry") } diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index 6f19abae2b9..f190b171363 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -22,8 +22,8 @@ func TestNewFileCache(t *testing.T) { ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - cache := NewCache[string](ctx, "test-component", 60, nil) - fc, ok := cache.(*FileCache[string]) + cache := NewCache(ctx, "test-component", 60, nil) + fc, ok := cache.impl.(*fileCache) require.True(t, ok) assert.True(t, strings.HasPrefix(fc.baseDir, cacheDir)) @@ -58,8 +58,8 @@ func TestNewFileCacheWithExistingDirectory(t *testing.T) { ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - cache := NewCache[string](ctx, "test-component", 60, nil) - fc, ok := cache.(*FileCache[string]) + cache := NewCache(ctx, "test-component", 60, nil) + fc, ok := cache.impl.(*fileCache) require.True(t, ok) require.NoError(t, err) assert.True(t, strings.HasPrefix(fc.baseDir, cacheDir)) @@ -76,8 +76,8 @@ func TestNewFileCacheInvalidPath(t *testing.T) { ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", invalidPath) - cache := NewCache[string](ctx, "test-component", 60, nil) - _, ok := cache.(*NoopFileCache[string]) + cache := NewCache(ctx, "test-component", 60, nil) + _, ok := cache.impl.(*noopFileCache) require.True(t, ok) } @@ -88,7 +88,7 @@ func TestFileCacheGetOrCompute(t *testing.T) { ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - cache := NewCache[string](ctx, "test-component", 60, nil) + cache := NewCache(ctx, "test-component", 60, nil) fingerprint := struct { Key string `json:"key"` @@ -101,7 +101,7 @@ func TestFileCacheGetOrCompute(t *testing.T) { // First call should compute the value var computeCalls int32 - result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + result, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return expectedValue, nil }) @@ -111,7 +111,7 @@ func TestFileCacheGetOrCompute(t *testing.T) { assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) // Second call should return cached value without computing - result2, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + result2, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "should-not-be-called", nil }) @@ -128,7 +128,7 @@ func TestFileCacheGetOrComputeError(t *testing.T) { ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - cache := NewCache[string](ctx, "test-component", 60, nil) + cache := NewCache(ctx, "test-component", 60, nil) fingerprint := struct { Key string `json:"key"` @@ -137,7 +137,7 @@ func TestFileCacheGetOrComputeError(t *testing.T) { } // Compute function returns error - result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + result, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { return "", assert.AnError }) @@ -153,7 +153,7 @@ func TestFileCacheGetOrComputeConcurrency(t *testing.T) { ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - cache := NewCache[string](ctx, "test-component", 60, nil) + cache := NewCache(ctx, "test-component", 60, nil) fingerprint := struct { Key string `json:"key"` @@ -170,7 +170,7 @@ func TestFileCacheGetOrComputeConcurrency(t *testing.T) { for range numGoroutines { go func() { - result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + result, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) time.Sleep(10 * time.Millisecond) // Simulate work return expectedValue, nil @@ -219,7 +219,7 @@ func TestFileCacheCleanupExpiredFiles(t *testing.T) { require.NoError(t, os.WriteFile(nonCacheFile, []byte("readme"), 0o644)) // Create cache - this should trigger cleanup - _, err := newFileCacheWithBaseDir[string](ctx, tempDir, expiryMinutes) + _, err := newFileCacheWithBaseDir(ctx, tempDir, expiryMinutes) require.NoError(t, err) // Check results @@ -236,11 +236,13 @@ func TestFileCacheCleanupExpiredFiles(t *testing.T) { func TestFileCacheInvalidJSON(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](ctx, tempDir, 60) + fc, err := newFileCacheWithBaseDir(ctx, tempDir, 60) require.NoError(t, err) // Enable cache for this test - cache.cacheEnabled = true + fc.cacheEnabled = true + + cache := &Cache{impl: fc} fingerprint := struct { Key string `json:"key"` @@ -251,13 +253,13 @@ func TestFileCacheInvalidJSON(t *testing.T) { // Manually write invalid JSON to the cache file cacheKey, err := fingerprintToHash(fingerprint) require.NoError(t, err) - cachePath := cache.getCachePath(cacheKey) + cachePath := fc.getCachePath(cacheKey) err = os.WriteFile(cachePath, []byte("invalid json {{{"), 0o600) require.NoError(t, err) // GetOrCompute should fail open and recompute when cache contains invalid JSON var computeCalls int32 - result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + result, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "recomputed-value", nil }) @@ -270,11 +272,13 @@ func TestFileCacheInvalidJSON(t *testing.T) { func TestFileCacheCorruptedData(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[int](ctx, tempDir, 60) + fc, err := newFileCacheWithBaseDir(ctx, tempDir, 60) require.NoError(t, err) // Enable cache for this test - cache.cacheEnabled = true + fc.cacheEnabled = true + + cache := &Cache{impl: fc} fingerprint := struct { Key string `json:"key"` @@ -285,13 +289,13 @@ func TestFileCacheCorruptedData(t *testing.T) { // Write valid JSON but wrong type (string instead of int) cacheKey, err := fingerprintToHash(fingerprint) require.NoError(t, err) - cachePath := cache.getCachePath(cacheKey) + cachePath := fc.getCachePath(cacheKey) err = os.WriteFile(cachePath, []byte(`"not-an-integer"`), 0o600) require.NoError(t, err) // GetOrCompute should fail open and recompute when cache type doesn't match var computeCalls int32 - result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (int, error) { + result, err := GetOrCompute[int](cache, ctx, fingerprint, func(ctx context.Context) (int, error) { atomic.AddInt32(&computeCalls, 1) return 42, nil }) @@ -304,17 +308,19 @@ func TestFileCacheCorruptedData(t *testing.T) { func TestFileCacheEmptyFingerprint(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](ctx, tempDir, 60) + fc, err := newFileCacheWithBaseDir(ctx, tempDir, 60) require.NoError(t, err) // Enable cache for this test - cache.cacheEnabled = true + fc.cacheEnabled = true + + cache := &Cache{impl: fc} // Empty struct fingerprint is valid fingerprint := struct{}{} var computeCalls int32 - result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + result, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "value", nil }) @@ -322,7 +328,7 @@ func TestFileCacheEmptyFingerprint(t *testing.T) { assert.Equal(t, "value", result) // Second call should use cache - result2, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + result2, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "should-not-be-called", nil }) @@ -334,11 +340,13 @@ func TestFileCacheEmptyFingerprint(t *testing.T) { func TestFileCacheMeasurementMode(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](ctx, tempDir, 60) + fc, err := newFileCacheWithBaseDir(ctx, tempDir, 60) require.NoError(t, err) // Keep cache disabled (measurement mode) - cache.cacheEnabled = false + fc.cacheEnabled = false + + cache := &Cache{impl: fc} fingerprint := struct { Key string `json:"key"` @@ -348,7 +356,7 @@ func TestFileCacheMeasurementMode(t *testing.T) { // First call var computeCalls int32 - result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + result, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "computed-value", nil }) @@ -357,7 +365,7 @@ func TestFileCacheMeasurementMode(t *testing.T) { assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) // Second call - in measurement mode, should always recompute - result2, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + result2, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "recomputed-value", nil }) @@ -381,11 +389,13 @@ func TestFileCacheReadPermissionError(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir[string](ctx, tempDir, 60) + fc, err := newFileCacheWithBaseDir(ctx, tempDir, 60) require.NoError(t, err) // Enable cache for this test - cache.cacheEnabled = true + fc.cacheEnabled = true + + cache := &Cache{impl: fc} fingerprint := struct { Key string `json:"key"` @@ -394,7 +404,7 @@ func TestFileCacheReadPermissionError(t *testing.T) { } // First, populate the cache - result, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + result, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { return "cached-value", nil }) require.NoError(t, err) @@ -412,7 +422,7 @@ func TestFileCacheReadPermissionError(t *testing.T) { // GetOrCompute should fail open and recompute when file is unreadable var computeCalls int32 - result2, err := cache.GetOrCompute(ctx, fingerprint, func(ctx context.Context) (string, error) { + result2, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "recomputed-value", nil }) diff --git a/libs/cache/noop_file_cache.go b/libs/cache/noop_file_cache.go index 9fe5cfe32b6..4cf4d26ef15 100644 --- a/libs/cache/noop_file_cache.go +++ b/libs/cache/noop_file_cache.go @@ -2,8 +2,9 @@ package cache import "context" -type NoopFileCache[T any] struct{} +type noopFileCache struct{} -func (c *NoopFileCache[T]) GetOrCompute(ctx context.Context, fingerprint any, compute func(ctx context.Context) (T, error)) (T, error) { - return compute(ctx) +func (c *noopFileCache) getOrComputeJSON(ctx context.Context, fingerprint any, compute func(ctx context.Context) ([]byte, error)) ([]byte, bool, error) { + result, err := compute(ctx) + return result, false, err } From 86dfd047d20ef7bfd1ff16347414f0ace306a788 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 10 Dec 2025 10:05:39 +0100 Subject: [PATCH 85/87] simplified --- libs/cache/cache.go | 20 +++++++------------- libs/cache/file_cache.go | 11 +++++------ libs/cache/noop_file_cache.go | 5 ++--- 3 files changed, 14 insertions(+), 22 deletions(-) diff --git a/libs/cache/cache.go b/libs/cache/cache.go index 42c6672b7fe..9b6e9831386 100644 --- a/libs/cache/cache.go +++ b/libs/cache/cache.go @@ -3,12 +3,13 @@ package cache import ( "context" "encoding/json" - "fmt" + + "github.com/databricks/cli/libs/log" ) // cacheImpl is the internal interface for cache implementations. type cacheImpl interface { - getOrComputeJSON(ctx context.Context, fingerprint any, compute func(ctx context.Context) ([]byte, error)) ([]byte, bool, error) + getOrComputeJSON(ctx context.Context, fingerprint any, compute func(ctx context.Context) ([]byte, error)) ([]byte, error) } // Cache provides a concrete cache that works with any type through the generic GetOrCompute function. @@ -37,7 +38,7 @@ func GetOrCompute[T any](c *Cache, ctx context.Context, fingerprint any, compute } // Call the internal method - jsonBytes, fromCache, err := c.impl.getOrComputeJSON(ctx, fingerprint, computeJSON) + jsonBytes, err := c.impl.getOrComputeJSON(ctx, fingerprint, computeJSON) if err != nil { return zero, err } @@ -45,16 +46,9 @@ func GetOrCompute[T any](c *Cache, ctx context.Context, fingerprint any, compute // Unmarshal into the target type var result T if err := json.Unmarshal(jsonBytes, &result); err != nil { - // If we got corrupted data from cache, fail open and recompute - if fromCache { - result, computeErr := compute(ctx) - if computeErr != nil { - return zero, computeErr - } - return result, nil - } - // If compute function returned invalid JSON, that's a real error - return zero, fmt.Errorf("failed to unmarshal computed data: %w", err) + // Fail open: if cached data is corrupted, log and recompute + log.Debugf(ctx, "[Local Cache] failed to unmarshal cached data, recomputing: %v", err) + return compute(ctx) } return result, nil diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 8b2ceb8439f..df47dd18b85 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -147,14 +147,13 @@ func (fc *fileCache) addTelemetryMetric(key string) { // Cache operations fail open: if caching fails, the compute function is still called. // When cacheEnabled is false, the cache checks if values exist and measures potential time savings, // but always computes and never returns cached values. -func (fc *fileCache) getOrComputeJSON(ctx context.Context, fingerprint any, compute func(ctx context.Context) ([]byte, error)) ([]byte, bool, error) { +func (fc *fileCache) getOrComputeJSON(ctx context.Context, fingerprint any, compute func(ctx context.Context) ([]byte, error)) ([]byte, error) { // Convert fingerprint to deterministic hash - this is our cache key cacheKey, err := fingerprintToHash(fingerprint) if err != nil { // Fail open: if we can't generate cache key, just compute directly log.Debugf(ctx, "[Local Cache] failed to generate cache key, computing without cache: %v", err) - result, err := compute(ctx) - return result, false, err + return compute(ctx) } log.Debugf(ctx, "[Local Cache] using cache key: %s", cacheKey) @@ -176,7 +175,7 @@ func (fc *fileCache) getOrComputeJSON(ctx context.Context, fingerprint any, comp // If cache is enabled, return the cached value if fc.cacheEnabled { - return cachedData, true, nil + return cachedData, nil } } else { log.Debugf(ctx, "[Local Cache] cache miss, computing") @@ -189,7 +188,7 @@ func (fc *fileCache) getOrComputeJSON(ctx context.Context, fingerprint any, comp if err != nil { log.Debugf(ctx, "[Local Cache] error while computing: %v", err) fc.addTelemetryMetric("local.cache.error") - return result, false, err + return result, err } // Record duration metrics @@ -203,7 +202,7 @@ func (fc *fileCache) getOrComputeJSON(ctx context.Context, fingerprint any, comp // Write to disk cache (failures are silent - cache write errors don't affect the result) fc.writeToCacheJSON(ctx, cachePath, result) - return result, false, nil + return result, nil } // readFromCacheJSON attempts to read data from the cache file. diff --git a/libs/cache/noop_file_cache.go b/libs/cache/noop_file_cache.go index 4cf4d26ef15..4b71be43fc0 100644 --- a/libs/cache/noop_file_cache.go +++ b/libs/cache/noop_file_cache.go @@ -4,7 +4,6 @@ import "context" type noopFileCache struct{} -func (c *noopFileCache) getOrComputeJSON(ctx context.Context, fingerprint any, compute func(ctx context.Context) ([]byte, error)) ([]byte, bool, error) { - result, err := compute(ctx) - return result, false, err +func (c *noopFileCache) getOrComputeJSON(ctx context.Context, fingerprint any, compute func(ctx context.Context) ([]byte, error)) ([]byte, error) { + return compute(ctx) } From cfdcd5d38a1dd304407c94c561a249aabbc97d47 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 10 Dec 2025 10:24:13 +0100 Subject: [PATCH 86/87] addressed feedback --- bundle/bundle.go | 14 ++---- bundle/config/mutator/initialize_cache.go | 3 +- .../config/mutator/populate_current_user.go | 2 +- libs/cache/cache.go | 2 +- libs/cache/file_cache.go | 29 +++++------ libs/cache/file_cache_env_test.go | 16 +++--- libs/cache/file_cache_expiry_test.go | 8 +-- libs/cache/file_cache_test.go | 50 +++++++++---------- 8 files changed, 60 insertions(+), 64 deletions(-) diff --git a/bundle/bundle.go b/bundle/bundle.go index 5102e739f40..837521f03f3 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -73,18 +73,10 @@ func (m *Metrics) AddBoolValue(key string, value bool) { m.BoolValues = append(m.BoolValues, protos.BoolMapEntry{Key: key, Value: value}) } -// SetDurationValue sets the value of a duration metric in milliseconds. -// If the metric does not exist, it is created. -// If the metric exists, it is updated. -// Ensures that the metric is unique. -func (m *Metrics) SetDurationValue(key string, value time.Duration) { +// AddDurationValue sets the value of a duration metric in milliseconds. +// The value is added to the list of measurements. +func (m *Metrics) AddDurationValue(key string, value time.Duration) { valueMs := value.Milliseconds() - for i, v := range m.LocalCacheMeasurementsMs { - if v.Key == key { - m.LocalCacheMeasurementsMs[i].Value = valueMs - return - } - } m.LocalCacheMeasurementsMs = append(m.LocalCacheMeasurementsMs, protos.IntMapEntry{Key: key, Value: valueMs}) } diff --git a/bundle/config/mutator/initialize_cache.go b/bundle/config/mutator/initialize_cache.go index b55ea5e29f8..d27e52efc12 100644 --- a/bundle/config/mutator/initialize_cache.go +++ b/bundle/config/mutator/initialize_cache.go @@ -2,6 +2,7 @@ package mutator import ( "context" + "time" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/cache" @@ -21,6 +22,6 @@ func (m *initializeCache) Name() string { func (m *initializeCache) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Initialize cache with 30 minute expiry for user information - b.Cache = cache.NewCache(ctx, "user", 30, &b.Metrics) + b.Cache = cache.NewCache(ctx, "user", 30*time.Minute, &b.Metrics) return nil } diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index ef654d87507..603df89b96d 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -33,7 +33,7 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. var err error fingerprint := b.GetUserFingerprint(ctx) - me, err = cache.GetOrCompute(b.Cache, ctx, fingerprint, func(ctx context.Context) (*iam.User, error) { + me, err = cache.GetOrCompute(ctx, b.Cache, fingerprint, func(ctx context.Context) (*iam.User, error) { currentUser, err := w.CurrentUser.Me(ctx) return currentUser, err }) diff --git a/libs/cache/cache.go b/libs/cache/cache.go index 9b6e9831386..0328fca8750 100644 --- a/libs/cache/cache.go +++ b/libs/cache/cache.go @@ -25,7 +25,7 @@ type Cache struct { // Cache operations fail open: if caching fails, the compute function is still called. // Returns an error only if the compute function fails. // The type parameter T is inferred from the compute function's return type. -func GetOrCompute[T any](c *Cache, ctx context.Context, fingerprint any, compute func(ctx context.Context) (T, error)) (T, error) { +func GetOrCompute[T any](ctx context.Context, c *Cache, fingerprint any, compute func(ctx context.Context) (T, error)) (T, error) { var zero T // Wrap the compute function to serialize to JSON diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index df47dd18b85..deb9780b65f 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -16,27 +16,27 @@ import ( // Metrics is a local interface for tracking cache telemetry. type Metrics interface { SetBoolValue(key string, value bool) - SetDurationValue(key string, value time.Duration) + AddDurationValue(key string, value time.Duration) } // fileCache implements the cacheImpl interface using local disk storage. type fileCache struct { - baseDir string - expiryMinutes int - mu sync.Mutex - metrics Metrics - cacheEnabled bool // If true, cached values are returned; if false, cache is only used for measurement + baseDir string + expiry time.Duration + mu sync.Mutex + metrics Metrics + cacheEnabled bool // If true, cached values are returned; if false, cache is only used for measurement } // newFileCacheWithBaseDir creates a new file-based cache that stores data in the specified directory. -func newFileCacheWithBaseDir(ctx context.Context, baseDir string, expiryMinutes int) (*fileCache, error) { +func newFileCacheWithBaseDir(ctx context.Context, baseDir string, expiry time.Duration) (*fileCache, error) { if err := os.MkdirAll(baseDir, 0o700); err != nil { return nil, fmt.Errorf("failed to create cache directory: %w", err) } fc := &fileCache{ - baseDir: baseDir, - expiryMinutes: expiryMinutes, + baseDir: baseDir, + expiry: expiry, } // Clean up expired files synchronously @@ -47,13 +47,13 @@ func newFileCacheWithBaseDir(ctx context.Context, baseDir string, expiryMinutes // isExpired checks if a file with the given modification time has expired. func (fc *fileCache) isExpired(modTime time.Time) bool { - expiryThreshold := time.Now().Add(-time.Duration(fc.expiryMinutes) * time.Minute) + expiryThreshold := time.Now().Add(-fc.expiry) return modTime.Before(expiryThreshold) } // cleanupExpiredFiles removes expired cache files from disk based on file modification time. // This runs synchronously once when the cache is created. -// Files older than expiryMinutes are deleted. +// Files older than expiry duration are deleted. func (fc *fileCache) cleanupExpiredFiles(ctx context.Context) { err := filepath.Walk(fc.baseDir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -116,7 +116,7 @@ func getCacheBaseDir(ctx context.Context) (string, error) { // - Always compute the value (never actually use the cache) // Set DATABRICKS_CACHE_ENABLED=true to enable actual caching. // The returned cache can handle multiple types through the generic GetOrCompute function. -func NewCache(ctx context.Context, component string, expiryMinutes int, metrics Metrics) *Cache { +func NewCache(ctx context.Context, component string, expiry time.Duration, metrics Metrics) *Cache { cacheBaseDir, err := getCacheBaseDir(ctx) if err != nil { return &Cache{impl: &noopFileCache{}} @@ -126,13 +126,14 @@ func NewCache(ctx context.Context, component string, expiryMinutes int, metrics // Sanitize version string for use in file paths version := build.GetInfo().GetSanitizedVersion() baseDir := filepath.Join(cacheBaseDir, version, component) - fc, err := newFileCacheWithBaseDir(ctx, baseDir, expiryMinutes) + fc, err := newFileCacheWithBaseDir(ctx, baseDir, expiry) if err != nil { return &Cache{impl: &noopFileCache{}} } fc.metrics = metrics // Check if cache is enabled; default is false (measurement-only mode) + // Only "true" enables caching; any other value (including "false", "1", etc.) keeps it disabled fc.cacheEnabled = env.Get(ctx, "DATABRICKS_CACHE_ENABLED") == "true" return &Cache{impl: fc} } @@ -194,7 +195,7 @@ func (fc *fileCache) getOrComputeJSON(ctx context.Context, fingerprint any, comp // Record duration metrics if fc.metrics != nil { computeDuration := time.Since(start) - fc.metrics.SetDurationValue("local.cache.compute_duration", computeDuration) + fc.metrics.AddDurationValue("local.cache.compute_duration", computeDuration) } log.Debugf(ctx, "[Local Cache] computed and stored result") diff --git a/libs/cache/file_cache_env_test.go b/libs/cache/file_cache_env_test.go index d1b205f2362..f8e5fb534e1 100644 --- a/libs/cache/file_cache_env_test.go +++ b/libs/cache/file_cache_env_test.go @@ -8,6 +8,7 @@ import ( "strings" "sync/atomic" "testing" + "time" "github.com/databricks/cli/libs/env" "github.com/stretchr/testify/assert" @@ -58,10 +59,11 @@ func TestCacheEnabledEnvVar(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // Create a unique subdirectory for this test testDir := filepath.Join(tempDir, tt.name) - fc, err := newFileCacheWithBaseDir(ctx, testDir, 60) + fc, err := newFileCacheWithBaseDir(ctx, testDir, 60*time.Minute) require.NoError(t, err) // Set cacheEnabled based on env var (simulate NewFileCache behavior) + // Only "true" enables caching; any other value keeps it disabled fc.cacheEnabled = env.Get(ctx, "DATABRICKS_CACHE_ENABLED") == "true" cache := &Cache{impl: fc} @@ -74,7 +76,7 @@ func TestCacheEnabledEnvVar(t *testing.T) { // First call - should always compute var computeCalls int32 - result, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { + result, err := GetOrCompute[string](ctx, cache, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "computed-value", nil }) @@ -83,7 +85,7 @@ func TestCacheEnabledEnvVar(t *testing.T) { assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) // Second call - should use cache only if enabled - result2, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { + result2, err := GetOrCompute[string](ctx, cache, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "should-not-be-called", nil }) @@ -110,7 +112,7 @@ func TestCacheDirEnvVar(t *testing.T) { customCacheDir := filepath.Join(tempDir, "custom-cache") t.Setenv("DATABRICKS_CACHE_DIR", customCacheDir) - cache := NewCache(ctx, "test-component", 60, nil) + cache := NewCache(ctx, "test-component", 60*time.Minute, nil) fc, ok := cache.impl.(*fileCache) require.True(t, ok) @@ -126,7 +128,7 @@ func TestCacheDirEnvVar(t *testing.T) { t.Run("uses default UserCacheDir when env var not set", func(t *testing.T) { os.Unsetenv("DATABRICKS_CACHE_DIR") - cache := NewCache(ctx, "test-component", 60, nil) + cache := NewCache(ctx, "test-component", 60*time.Minute, nil) fc, ok := cache.impl.(*fileCache) require.True(t, ok) @@ -146,7 +148,7 @@ func TestCacheDirEnvVar(t *testing.T) { // Set an invalid path (no permissions) t.Setenv("DATABRICKS_CACHE_DIR", "/root/invalid-cache-dir") - cache := NewCache(ctx, "test-component", 60, nil) + cache := NewCache(ctx, "test-component", 60*time.Minute, nil) _, ok := cache.impl.(*noopFileCache) require.True(t, ok) }) @@ -158,7 +160,7 @@ func TestCacheIsolationByVersion(t *testing.T) { t.Setenv("DATABRICKS_CACHE_DIR", tempDir) // Create cache for component - cache := NewCache(ctx, "test-component", 60, nil) + cache := NewCache(ctx, "test-component", 60*time.Minute, nil) fc, ok := cache.impl.(*fileCache) require.True(t, ok) diff --git a/libs/cache/file_cache_expiry_test.go b/libs/cache/file_cache_expiry_test.go index d21f09946da..6ae55888de9 100644 --- a/libs/cache/file_cache_expiry_test.go +++ b/libs/cache/file_cache_expiry_test.go @@ -17,7 +17,7 @@ func TestFileCacheExpiryBehavior(t *testing.T) { tempDir := t.TempDir() // Create cache with 1 minute expiry - fc, err := newFileCacheWithBaseDir(ctx, tempDir, 1) + fc, err := newFileCacheWithBaseDir(ctx, tempDir, 1*time.Minute) require.NoError(t, err) // Enable cache for this test (default is measurement-only mode) @@ -32,7 +32,7 @@ func TestFileCacheExpiryBehavior(t *testing.T) { } // Compute and store a value - result, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { + result, err := GetOrCompute[string](ctx, cache, fingerprint, func(ctx context.Context) (string, error) { return "test-value", nil }) require.NoError(t, err) @@ -60,7 +60,7 @@ func TestFileCacheExpiryBehavior(t *testing.T) { // Verify GetOrCompute treats it as a cache miss and recomputes callCount := 0 - result, err = GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { + result, err = GetOrCompute[string](ctx, cache, fingerprint, func(ctx context.Context) (string, error) { callCount++ return "recomputed-value", nil }) @@ -73,7 +73,7 @@ func TestFileCacheExpiryBehavior(t *testing.T) { func TestReadFromCacheRespectsExpiry(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - cache, err := newFileCacheWithBaseDir(ctx, tempDir, 1) // 1 minute expiry + cache, err := newFileCacheWithBaseDir(ctx, tempDir, 1*time.Minute) // 1 minute expiry require.NoError(t, err) // Create an expired cache file by setting its mtime to 2 hours ago diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index f190b171363..214c400e7fe 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -22,7 +22,7 @@ func TestNewFileCache(t *testing.T) { ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - cache := NewCache(ctx, "test-component", 60, nil) + cache := NewCache(ctx, "test-component", 60*time.Minute, nil) fc, ok := cache.impl.(*fileCache) require.True(t, ok) assert.True(t, strings.HasPrefix(fc.baseDir, cacheDir)) @@ -58,7 +58,7 @@ func TestNewFileCacheWithExistingDirectory(t *testing.T) { ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - cache := NewCache(ctx, "test-component", 60, nil) + cache := NewCache(ctx, "test-component", 60*time.Minute, nil) fc, ok := cache.impl.(*fileCache) require.True(t, ok) require.NoError(t, err) @@ -76,7 +76,7 @@ func TestNewFileCacheInvalidPath(t *testing.T) { ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", invalidPath) - cache := NewCache(ctx, "test-component", 60, nil) + cache := NewCache(ctx, "test-component", 60*time.Minute, nil) _, ok := cache.impl.(*noopFileCache) require.True(t, ok) } @@ -88,7 +88,7 @@ func TestFileCacheGetOrCompute(t *testing.T) { ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - cache := NewCache(ctx, "test-component", 60, nil) + cache := NewCache(ctx, "test-component", 60*time.Minute, nil) fingerprint := struct { Key string `json:"key"` @@ -101,7 +101,7 @@ func TestFileCacheGetOrCompute(t *testing.T) { // First call should compute the value var computeCalls int32 - result, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { + result, err := GetOrCompute[string](ctx, cache, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return expectedValue, nil }) @@ -111,7 +111,7 @@ func TestFileCacheGetOrCompute(t *testing.T) { assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) // Second call should return cached value without computing - result2, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { + result2, err := GetOrCompute[string](ctx, cache, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "should-not-be-called", nil }) @@ -128,7 +128,7 @@ func TestFileCacheGetOrComputeError(t *testing.T) { ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - cache := NewCache(ctx, "test-component", 60, nil) + cache := NewCache(ctx, "test-component", 60*time.Minute, nil) fingerprint := struct { Key string `json:"key"` @@ -137,7 +137,7 @@ func TestFileCacheGetOrComputeError(t *testing.T) { } // Compute function returns error - result, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { + result, err := GetOrCompute[string](ctx, cache, fingerprint, func(ctx context.Context) (string, error) { return "", assert.AnError }) @@ -153,7 +153,7 @@ func TestFileCacheGetOrComputeConcurrency(t *testing.T) { ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - cache := NewCache(ctx, "test-component", 60, nil) + cache := NewCache(ctx, "test-component", 60*time.Minute, nil) fingerprint := struct { Key string `json:"key"` @@ -170,7 +170,7 @@ func TestFileCacheGetOrComputeConcurrency(t *testing.T) { for range numGoroutines { go func() { - result, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { + result, err := GetOrCompute[string](ctx, cache, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) time.Sleep(10 * time.Millisecond) // Simulate work return expectedValue, nil @@ -198,7 +198,7 @@ func TestFileCacheGetOrComputeConcurrency(t *testing.T) { func TestFileCacheCleanupExpiredFiles(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - expiryMinutes := 60 + expiry := 60 * time.Minute // Create some cache files manually - one expired, one valid now := time.Now() @@ -219,7 +219,7 @@ func TestFileCacheCleanupExpiredFiles(t *testing.T) { require.NoError(t, os.WriteFile(nonCacheFile, []byte("readme"), 0o644)) // Create cache - this should trigger cleanup - _, err := newFileCacheWithBaseDir(ctx, tempDir, expiryMinutes) + _, err := newFileCacheWithBaseDir(ctx, tempDir, expiry) require.NoError(t, err) // Check results @@ -236,7 +236,7 @@ func TestFileCacheCleanupExpiredFiles(t *testing.T) { func TestFileCacheInvalidJSON(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - fc, err := newFileCacheWithBaseDir(ctx, tempDir, 60) + fc, err := newFileCacheWithBaseDir(ctx, tempDir, 60*time.Minute) require.NoError(t, err) // Enable cache for this test @@ -259,7 +259,7 @@ func TestFileCacheInvalidJSON(t *testing.T) { // GetOrCompute should fail open and recompute when cache contains invalid JSON var computeCalls int32 - result, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { + result, err := GetOrCompute[string](ctx, cache, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "recomputed-value", nil }) @@ -272,7 +272,7 @@ func TestFileCacheInvalidJSON(t *testing.T) { func TestFileCacheCorruptedData(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - fc, err := newFileCacheWithBaseDir(ctx, tempDir, 60) + fc, err := newFileCacheWithBaseDir(ctx, tempDir, 60*time.Minute) require.NoError(t, err) // Enable cache for this test @@ -295,7 +295,7 @@ func TestFileCacheCorruptedData(t *testing.T) { // GetOrCompute should fail open and recompute when cache type doesn't match var computeCalls int32 - result, err := GetOrCompute[int](cache, ctx, fingerprint, func(ctx context.Context) (int, error) { + result, err := GetOrCompute[int](ctx, cache, fingerprint, func(ctx context.Context) (int, error) { atomic.AddInt32(&computeCalls, 1) return 42, nil }) @@ -308,7 +308,7 @@ func TestFileCacheCorruptedData(t *testing.T) { func TestFileCacheEmptyFingerprint(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - fc, err := newFileCacheWithBaseDir(ctx, tempDir, 60) + fc, err := newFileCacheWithBaseDir(ctx, tempDir, 60*time.Minute) require.NoError(t, err) // Enable cache for this test @@ -320,7 +320,7 @@ func TestFileCacheEmptyFingerprint(t *testing.T) { fingerprint := struct{}{} var computeCalls int32 - result, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { + result, err := GetOrCompute[string](ctx, cache, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "value", nil }) @@ -328,7 +328,7 @@ func TestFileCacheEmptyFingerprint(t *testing.T) { assert.Equal(t, "value", result) // Second call should use cache - result2, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { + result2, err := GetOrCompute[string](ctx, cache, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "should-not-be-called", nil }) @@ -340,7 +340,7 @@ func TestFileCacheEmptyFingerprint(t *testing.T) { func TestFileCacheMeasurementMode(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - fc, err := newFileCacheWithBaseDir(ctx, tempDir, 60) + fc, err := newFileCacheWithBaseDir(ctx, tempDir, 60*time.Minute) require.NoError(t, err) // Keep cache disabled (measurement mode) @@ -356,7 +356,7 @@ func TestFileCacheMeasurementMode(t *testing.T) { // First call var computeCalls int32 - result, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { + result, err := GetOrCompute[string](ctx, cache, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "computed-value", nil }) @@ -365,7 +365,7 @@ func TestFileCacheMeasurementMode(t *testing.T) { assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) // Second call - in measurement mode, should always recompute - result2, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { + result2, err := GetOrCompute[string](ctx, cache, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "recomputed-value", nil }) @@ -389,7 +389,7 @@ func TestFileCacheReadPermissionError(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - fc, err := newFileCacheWithBaseDir(ctx, tempDir, 60) + fc, err := newFileCacheWithBaseDir(ctx, tempDir, 60*time.Minute) require.NoError(t, err) // Enable cache for this test @@ -404,7 +404,7 @@ func TestFileCacheReadPermissionError(t *testing.T) { } // First, populate the cache - result, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { + result, err := GetOrCompute[string](ctx, cache, fingerprint, func(ctx context.Context) (string, error) { return "cached-value", nil }) require.NoError(t, err) @@ -422,7 +422,7 @@ func TestFileCacheReadPermissionError(t *testing.T) { // GetOrCompute should fail open and recompute when file is unreadable var computeCalls int32 - result2, err := GetOrCompute[string](cache, ctx, fingerprint, func(ctx context.Context) (string, error) { + result2, err := GetOrCompute[string](ctx, cache, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "recomputed-value", nil }) From 07c05b62a0b064193361470dbfb1b5401774e366 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 10 Dec 2025 11:48:23 +0100 Subject: [PATCH 87/87] addressed comments --- .../config/mutator/populate_current_user.go | 8 +-- internal/build/info.go | 2 +- internal/build/info_test.go | 49 +++++++++++++++++++ libs/cache/cache.go | 3 ++ libs/cache/file_cache_clear.go | 12 ++++- libs/cache/file_cache_env_test.go | 22 ++++----- 6 files changed, 75 insertions(+), 21 deletions(-) diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index 603df89b96d..0088a024516 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -29,13 +29,9 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. } w := b.WorkspaceClient() - var me *iam.User - var err error - fingerprint := b.GetUserFingerprint(ctx) - me, err = cache.GetOrCompute(ctx, b.Cache, fingerprint, func(ctx context.Context) (*iam.User, error) { - currentUser, err := w.CurrentUser.Me(ctx) - return currentUser, err + me, err := cache.GetOrCompute(ctx, b.Cache, fingerprint, func(ctx context.Context) (*iam.User, error) { + return w.CurrentUser.Me(ctx) }) if err != nil { return diag.FromErr(err) diff --git a/internal/build/info.go b/internal/build/info.go index 3f03c6f6a7c..15967be8ff4 100644 --- a/internal/build/info.go +++ b/internal/build/info.go @@ -30,7 +30,7 @@ type Info struct { BuildTime time.Time } -// sanitizeVersion removes characters from version string that might be problematic in file paths. +// GetSanitizedVersion removes characters from version string that might be problematic in file paths. // Particularly important for Windows which has restrictions on certain characters. func (i Info) GetSanitizedVersion() string { // Replace + with - (used in version metadata like "1.0.0+abc123") diff --git a/internal/build/info_test.go b/internal/build/info_test.go index 1ae94fbce75..7b33a114e86 100644 --- a/internal/build/info_test.go +++ b/internal/build/info_test.go @@ -2,8 +2,57 @@ package build import ( "testing" + + "github.com/stretchr/testify/assert" ) func TestGetDetails(t *testing.T) { GetInfo() } + +func TestGetSanitizedVersion(t *testing.T) { + tests := []struct { + name string + version string + expected string + }{ + { + name: "version with plus", + version: "1.0.0+abc123", + expected: "1.0.0-abc123", + }, + { + name: "version with colon (Windows problematic)", + version: "1.0.0:dev", + expected: "1.0.0-dev", + }, + { + name: "version with forward slash (Windows problematic)", + version: "1.0.0/beta", + expected: "1.0.0-beta", + }, + { + name: "version with backslash (Windows problematic)", + version: "1.0.0\\test", + expected: "1.0.0-test", + }, + { + name: "version with multiple problematic characters", + version: "1.0.0+abc:123/test\\dev", + expected: "1.0.0-abc-123-test-dev", + }, + { + name: "clean version", + version: "1.0.0-dev", + expected: "1.0.0-dev", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + info := Info{Version: tt.version} + result := info.GetSanitizedVersion() + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/libs/cache/cache.go b/libs/cache/cache.go index 0328fca8750..513f7ebd002 100644 --- a/libs/cache/cache.go +++ b/libs/cache/cache.go @@ -9,6 +9,9 @@ import ( // cacheImpl is the internal interface for cache implementations. type cacheImpl interface { + // getOrComputeJSON retrieves cached JSON bytes or computes them. + // The compute function must return JSON-encoded data as []byte. + // The returned []byte is also expected to be JSON-encoded. getOrComputeJSON(ctx context.Context, fingerprint any, compute func(ctx context.Context) ([]byte, error)) ([]byte, error) } diff --git a/libs/cache/file_cache_clear.go b/libs/cache/file_cache_clear.go index 01ca6d15244..d4a51647e23 100644 --- a/libs/cache/file_cache_clear.go +++ b/libs/cache/file_cache_clear.go @@ -7,8 +7,16 @@ import ( // ClearFileCache removes all cached files from the Databricks cache directory. // This clears the cache for ALL CLI versions, not just the current version. -// The cache is organized as: /// -// This function removes the entire directory. +// +// The cache directory structure is: +// +// ~/.cache/databricks/ (or %LOCALAPPDATA%\databricks\ on Windows) +// └── / +// └── / +// ├── .json +// └── ... +// +// This function removes the entire databricks cache directory (all versions and components). // Returns the path of the cleared directory on success. func ClearFileCache(ctx context.Context) (string, error) { databricksCacheDir, err := getCacheBaseDir(ctx) diff --git a/libs/cache/file_cache_env_test.go b/libs/cache/file_cache_env_test.go index f8e5fb534e1..a2e2db4aed7 100644 --- a/libs/cache/file_cache_env_test.go +++ b/libs/cache/file_cache_env_test.go @@ -52,21 +52,19 @@ func TestCacheEnabledEnvVar(t *testing.T) { } for _, tt := range tests { - // Set up environment - if tt.envValue != "" { - t.Setenv("DATABRICKS_CACHE_ENABLED", tt.envValue) - } t.Run(tt.name, func(t *testing.T) { // Create a unique subdirectory for this test testDir := filepath.Join(tempDir, tt.name) - fc, err := newFileCacheWithBaseDir(ctx, testDir, 60*time.Minute) - require.NoError(t, err) - // Set cacheEnabled based on env var (simulate NewFileCache behavior) - // Only "true" enables caching; any other value keeps it disabled - fc.cacheEnabled = env.Get(ctx, "DATABRICKS_CACHE_ENABLED") == "true" + // Set up context with environment variable + testCtx := ctx + if tt.envValue != "" { + testCtx = env.Set(testCtx, "DATABRICKS_CACHE_ENABLED", tt.envValue) + } + testCtx = env.Set(testCtx, "DATABRICKS_CACHE_DIR", testDir) - cache := &Cache{impl: fc} + // Use NewCache to properly initialize the cache + cache := NewCache(testCtx, "test-component", 60*time.Minute, nil) fingerprint := struct { Key string `json:"key"` @@ -76,7 +74,7 @@ func TestCacheEnabledEnvVar(t *testing.T) { // First call - should always compute var computeCalls int32 - result, err := GetOrCompute[string](ctx, cache, fingerprint, func(ctx context.Context) (string, error) { + result, err := GetOrCompute[string](testCtx, cache, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "computed-value", nil }) @@ -85,7 +83,7 @@ func TestCacheEnabledEnvVar(t *testing.T) { assert.Equal(t, int32(1), atomic.LoadInt32(&computeCalls)) // Second call - should use cache only if enabled - result2, err := GetOrCompute[string](ctx, cache, fingerprint, func(ctx context.Context) (string, error) { + result2, err := GetOrCompute[string](testCtx, cache, fingerprint, func(ctx context.Context) (string, error) { atomic.AddInt32(&computeCalls, 1) return "should-not-be-called", nil })