Add template renderer for Databricks templates

libs/template/helpers.go

@@ -0,0 +1,20 @@
+package template
+
+import (
+	"fmt"
+	"text/template"
+)
+
+type ErrFail struct {
+	msg string
+}
+
+func (err ErrFail) Error() string {
+	return err.msg
+}
+
+var helperFuncs = template.FuncMap{
+	"fail": func(format string, args ...any) (any, error) {
+ return nil, ErrFail{fmt.Sprintf(format, args...)}
+	},
+}

libs/template/renderer.go

@@ -0,0 +1,316 @@
+package template
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/databricks/cli/libs/filer"
+	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/databricks-sdk-go/logger"
+	"golang.org/x/exp/slices"
+)
+
+type inMemoryFile struct {
+	// Root path for the project instance. This path uses the system's default
+	// file separator. For example /foo/bar on Unix and C:\foo\bar on windows
+	root string
+
+	// Unix like relPath for the file (using '/' as the separator). This path
+	// is relative to the root. Using unix like relative paths enables skip patterns
+	// to work across both windows and unix based operating systems.
+	relPath string
+	content []byte
+	perm fs.FileMode
+}
+
+func (f *inMemoryFile) fullPath() string {
+	return filepath.Join(f.root, filepath.FromSlash(f.relPath))
+}
+
+func (f *inMemoryFile) persistToDisk() error {
+	path := f.fullPath()
+
+	err := os.MkdirAll(filepath.Dir(path), 0755)
+	if err != nil {
+ return err
+	}
+	return os.WriteFile(path, f.content, f.perm)
+}
+
+// Renders a databricks template as a project
+type renderer struct {
+	ctx context.Context
+
+	// A config that is the "dot" value available to any template being rendered.
+	// Refer to https://pkg.go.dev/text/template for how templates can use
+	// this "dot" value
+	config map[string]any
+
+	// A base template with helper functions and user defined templates in the
+	// library directory loaded. This is cloned for each project template computation
+	// during file tree walk
+	baseTemplate *template.Template
+
+	// List of in memory files generated from template
+	files []*inMemoryFile
+
+	// Glob patterns for files and directories to skip. There are three possible
+	// outcomes for skip:
+	//
+	// 1. File is not generated. This happens if one of the file's parent directories
+	// match a glob pattern
+	//
+	// 2. File is generated but not persisted to disk. This happens if the file itself
+	// matches a glob pattern, but none of it's parents match a glob pattern from the list
+	//
+	// 3. File is persisted to disk. This happens if the file and it's parent directories
+	// do not match any glob patterns from this list
+	skipPatterns []string
+
+	// Filer rooted at template root. The file tree from this root is walked to
+	// generate the project
+	templateFiler filer.Filer
+
+	// Root directory for the project instantiated from the template
+	instanceRoot string
+}
+
+func newRenderer(ctx context.Context, config map[string]any, templateRoot, libraryRoot, instanceRoot string) (*renderer, error) {
+	// Initialize new template, with helper functions loaded
+	tmpl := template.New("").Funcs(helperFuncs)
+
+	// Load user defined associated templates from the library root
+	libraryGlob := filepath.Join(libraryRoot, "*")
+	matches, err := filepath.Glob(libraryGlob)
+	if err != nil {
+ return nil, err
+	}
+	if len(matches) != 0 {
+ tmpl, err = tmpl.ParseFiles(matches...)
+ if err != nil {
+ return nil, err
+ }
+	}
+
+	templateFiler, err := filer.NewLocalClient(templateRoot)
+	if err != nil {
+ return nil, err
+	}
+
+	ctx = log.NewContext(ctx, log.GetLogger(ctx).With("action", "initialize-template"))
+
+	return &renderer{
+ ctx: ctx,
+ config: config,
+ baseTemplate: tmpl,
+ files: make([]*inMemoryFile, 0),
+ skipPatterns: make([]string, 0),
+ templateFiler: templateFiler,
+ instanceRoot: instanceRoot,
+	}, nil
+}
+
+// Executes the template by applying config on it. Returns the materialized template
+// as a string
+func (r *renderer) executeTemplate(templateDefinition string) (string, error) {
+	// Create copy of base template so as to not overwrite it
+	tmpl, err := r.baseTemplate.Clone()
+	if err != nil {
+ return "", err
+	}
+
+	// Parse the template text
+	tmpl, err = tmpl.Parse(templateDefinition)
+	if err != nil {
+ return "", err
+	}
+
+	// Execute template and get result
+	result := strings.Builder{}
+	err = tmpl.Execute(&result, r.config)
+	if err != nil {
+ return "", err
+	}
+	return result.String(), nil
+}
+
+func (r *renderer) computeFile(relPathTemplate string) (*inMemoryFile, error) {
+	// read template file contents
+	templateReader, err := r.templateFiler.Read(r.ctx, relPathTemplate)
+	if err != nil {
+ return nil, err
+	}
+	contentTemplate, err := io.ReadAll(templateReader)
+	if err != nil {
+ return nil, err
+	}
+
+	// read file permissions
+	info, err := r.templateFiler.Stat(r.ctx, relPathTemplate)
+	if err != nil {
+ return nil, err
+	}
+	perm := info.Mode().Perm()
+
+	// execute the contents of the file as a template
+	content, err := r.executeTemplate(string(contentTemplate))
+	// Capture errors caused by the "fail" helper function
+	if target := (&ErrFail{}); errors.As(err, target) {
+ return nil, target
+	}
+	if err != nil {
+ return nil, fmt.Errorf("failed to compute file content for %s. %w", relPathTemplate, err)
+	}
+
+	// Execute relative path template to get materialized path for the file
+	relPath, err := r.executeTemplate(relPathTemplate)
+	if err != nil {
+ return nil, err
+	}
+
+	return &inMemoryFile{
+ root: r.instanceRoot,
+ relPath: relPath,
+ content: []byte(content),
+ perm: perm,
+	}, nil
+}
+
+
+// This function walks the template file tree to generate an in memory representation
+// of a project.
+//
+// During file tree walk, in the current directory, we would like to determine
+// all possible {{skip}} function calls before we process any of the directories
+// so that we can skip them eagerly if needed. That is in the current working directory
+// we would like to process all files before we process any of the directories.
+//
+// This is not possible using the std library WalkDir which processes the files in
+// lexical order which is why this function implements BFS.
+func (r *renderer) walk() error {
+	directories := []string{"."}
+	var currentDirectory string
+
+	for len(directories) > 0 {
+ currentDirectory, directories = directories[0], directories[1:]
+
+ // Skip current directory if it matches any of accumulated skip patterns
+ instanceDirectory, err := r.executeTemplate(currentDirectory)
+ if err != nil {
+ return err
+ }
+ isSkipped, err := r.isSkipped(instanceDirectory)
+ if err != nil {
+ return err
+ }
+ if isSkipped {
+ logger.Infof(r.ctx, "skipping directory: %s", instanceDirectory)
+ continue
+ }
+
+ // Add skip function, which accumulates skip patterns relative to current
+ // directory
+ r.baseTemplate.Funcs(template.FuncMap{
+ "skip": func(relPattern string) string {
+ // patterns are specified relative to current directory of the file
+ // the {{skip}} function is called from.
+ pattern := path.Join(currentDirectory, relPattern)
+ if !slices.Contains(r.skipPatterns, pattern) {
+ logger.Infof(r.ctx, "adding skip pattern: %s", pattern)
+ r.skipPatterns = append(r.skipPatterns, pattern)
+ }
+ // return empty string will print nothing at function call site
+ // when executing the template
+ return ""
+ },
+ })
+
+ // Process all entries in current directory
+ //
+ // 1. For files: the templates in the file name and content are executed, and
+ // a in memory representation of the file is generated
+ //
+ // 2. For directories: They are appended to a slice, which acts as a queue
+ // allowing BFS traversal of the template file tree
+ entries, err := r.templateFiler.ReadDir(r.ctx, currentDirectory)
+ if err != nil {
+ return err
+ }
+ for _, entry := range entries {
+ if entry.IsDir() {
+ // Add to slice, for BFS traversal
+ directories = append(directories, path.Join(currentDirectory, entry.Name()))
+ continue
+ }
+
+ // Generate in memory representation of file
+ f, err := r.computeFile(path.Join(currentDirectory, entry.Name()))
+ if err != nil {
+ return err
+ }
+ logger.Infof(r.ctx, "added file to list of in memory files: %s", f.relPath)
+ r.files = append(r.files, f)
+ }
+
+	}
+	return nil
+}
+
+func (r *renderer) persistToDisk() error {
+	// Accumulate files which we will persist, skipping files whose path matches
+	// any of the skip patterns
+	filesToPersist := make([]*inMemoryFile, 0)
+	for _, file := range r.files {
+ isSkipped, err := r.isSkipped(file.relPath)
+ if err != nil {
+ return err
+ }
+ if isSkipped {
+ log.Infof(r.ctx, "skipping file: %s", file.relPath)
+ continue
+ }
+ filesToPersist = append(filesToPersist, file)
+	}
+
+	// Assert no conflicting files exist
+	for _, file := range filesToPersist {
+ path := file.fullPath()
+ _, err := os.Stat(path)
+ if err == nil {
+ return fmt.Errorf("failed to persist to disk, conflict with existing file: %s", path)
+ }
+ if err != nil && !os.IsNotExist(err) {
+ return fmt.Errorf("error while verifying file %s does not already exist: %w", path, err)
+ }
+	}
+
+	// Persist files to disk
+	for _, file := range filesToPersist {
+ err := file.persistToDisk()
+ if err != nil {
+ return err
+ }
+	}
+	return nil
+}
+
+func (r *renderer) isSkipped(filePath string) (bool, error) {
+	for _, pattern := range r.skipPatterns {
+ isMatch, err := path.Match(pattern, filePath)
+ if err != nil {
+ return false, err
+ }
+ if isMatch {
+ return true, nil
+ }
+	}
+	return false, nil
+}