Custom annotations for bundle-specific JSON schema fields

.codegen.json

@@ -11,7 +11,7 @@
  "required": ["go"],
  "post_generate": [
  "go test -timeout 240s -run TestConsistentDatabricksSdkVersion github.com/databricks/cli/internal/build",
- "go run ./bundle/internal/schema/*.go ./bundle/schema/jsonschema.json",
+ "make schema",
  "echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes",
  "echo 'go.sum linguist-generated=true' >> ./.gitattributes",
  "echo 'bundle/schema/jsonschema.json linguist-generated=true' >> ./.gitattributes"

.github/workflows/push.yml

@@ -99,14 +99,19 @@ jobs:
  # By default the ajv-cli runs in strict mode which will fail if the schema
  # itself is not valid. Strict mode is more strict than the JSON schema
  # specification. See for details: https://ajv.js.org/options.html#strict-mode-options
+ # The ajv-cli is configured to use the markdownDescription keyword which is not part of the JSON schema specification,
+ # but is used in editors like VSCode to render markdown in the description field
  - name: Validate bundle schema
  run: |
  go run main.go bundle schema > schema.json
 
+ # Add markdownDescription keyword to ajv
+ echo "module.exports=function(a){a.addKeyword('markdownDescription')}" >> keywords.js
+
  for file in ./bundle/internal/schema/testdata/pass/*.yml; do
- ajv test -s schema.json -d $file --valid
+ ajv test -s schema.json -d $file --valid -c=./keywords.js
  done
 
  for file in ./bundle/internal/schema/testdata/fail/*.yml; do
- ajv test -s schema.json -d $file --invalid
+ ajv test -s schema.json -d $file --invalid -c=./keywords.js
  done

Makefile

@@ -29,6 +29,10 @@ snapshot:
 vendor:
 	@echo "✓ Filling vendor folder with library code ..."
 	@go mod vendor
+
+schema:
+	@echo "✓ Generating json-schema ..."
+	@go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json
 
 INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./integration/..." -- -parallel 4 -timeout=2h
 
@@ -38,4 +42,4 @@ integration:
 integration-short:
 	$(INTEGRATION) -short
 
-.PHONY: lint lintcheck test testonly coverage build snapshot vendor integration integration-short
+.PHONY: lint lintcheck test testonly coverage build snapshot vendor schema integration integration-short

bundle/internal/schema/annotations.go

@@ -0,0 +1,209 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"reflect"
+	"regexp"
+	"strings"
+
+	yaml3 "gopkg.in/yaml.v3"
+
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/convert"
+	"github.com/databricks/cli/libs/dyn/merge"
+	"github.com/databricks/cli/libs/dyn/yamlloader"
+	"github.com/databricks/cli/libs/dyn/yamlsaver"
+	"github.com/databricks/cli/libs/jsonschema"
+)
+
+type annotation struct {
+	Description string `json:"description,omitempty"`
+	MarkdownDescription string `json:"markdown_description,omitempty"`
+	Title string `json:"title,omitempty"`
+	Default any `json:"default,omitempty"`
+	Enum []any `json:"enum,omitempty"`
+}
+
+type annotationHandler struct {
+	// Annotations read from all annotation files including all overrides
+	parsedAnnotations annotationFile
+	// Missing annotations for fields that are found in config that need to be added to the annotation file
+	missingAnnotations annotationFile
+}
+
+/**
+ * Parsed file with annotations, expected format:
+ * github.com/databricks/cli/bundle/config.Bundle:
+ * cluster_id:
+ * description: "Description"
+ */
+type annotationFile map[string]map[string]annotation
+
+const Placeholder = "PLACEHOLDER"
+
+// Adds annotations to the JSON schema reading from the annotation files.
+// More details https://json-schema.org/understanding-json-schema/reference/annotations
+func newAnnotationHandler(sources []string) (*annotationHandler, error) {
+	prev := dyn.NilValue
+	for _, path := range sources {
+ b, err := os.ReadFile(path)
+ if err != nil {
+ return nil, err
+ }
+ generated, err := yamlloader.LoadYAML(path, bytes.NewBuffer(b))
+ if err != nil {
+ return nil, err
+ }
+ prev, err = merge.Merge(prev, generated)
+ if err != nil {
+ return nil, err
+ }
+	}
+
+	var data annotationFile
+
+	err := convert.ToTyped(&data, prev)
+	if err != nil {
+ return nil, err
+	}
+
+	d := &annotationHandler{}
+	d.parsedAnnotations = data
+	d.missingAnnotations = annotationFile{}
+	return d, nil
+}
+
+func (d *annotationHandler) addAnnotations(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
+	refPath := getPath(typ)
+	shouldHandle := strings.HasPrefix(refPath, "github.com")
+	if !shouldHandle {
+ return s
+	}
+
+	annotations := d.parsedAnnotations[refPath]
+	if annotations == nil {
+ annotations = map[string]annotation{}
+	}
+
+	rootTypeAnnotation, ok := annotations[RootTypeKey]
+	if ok {
+ assignAnnotation(&s, rootTypeAnnotation)
+	}
+
+	for k, v := range s.Properties {
+ item := annotations[k]
+ if item.Description == "" {
+ item.Description = Placeholder
+
+ emptyAnnotations := d.missingAnnotations[refPath]
+ if emptyAnnotations == nil {
+ emptyAnnotations = map[string]annotation{}
+ d.missingAnnotations[refPath] = emptyAnnotations
+ }
+ emptyAnnotations[k] = item
+ }
+ assignAnnotation(v, item)
+	}
+	return s
+}
+
+// Writes missing annotations with placeholder values back to the annotation file
+func (d *annotationHandler) syncWithMissingAnnotations(outputPath string) error {
+	existingFile, err := os.ReadFile(outputPath)
+	if err != nil {
+ return err
+	}
+	existing, err := yamlloader.LoadYAML("", bytes.NewBuffer(existingFile))
+	if err != nil {
+ return err
+	}
+	missingAnnotations, err := convert.FromTyped(&d.missingAnnotations, dyn.NilValue)
+	if err != nil {
+ return err
+	}
+
+	output, err := merge.Merge(existing, missingAnnotations)
+	if err != nil {
+ return err
+	}
+
+	err = saveYamlWithStyle(outputPath, output)
+	if err != nil {
+ return err
+	}
+	return nil
+}
+
+func getPath(typ reflect.Type) string {
+	return typ.PkgPath() + "." + typ.Name()
+}
+
+func assignAnnotation(s *jsonschema.Schema, a annotation) {
+	if a.Description != Placeholder {
+ s.Description = a.Description
+	}
+
+	if a.Default != nil {
+ s.Default = a.Default
+	}
+	s.MarkdownDescription = convertLinksToAbsoluteUrl(a.MarkdownDescription)
+	s.Title = a.Title
+	s.Enum = a.Enum
+}
+
+func saveYamlWithStyle(outputPath string, input dyn.Value) error {
+	style := map[string]yaml3.Style{}
+	file, _ := input.AsMap()
+	for _, v := range file.Keys() {
+ style[v.MustString()] = yaml3.LiteralStyle
+	}
+
+	saver := yamlsaver.NewSaverWithStyle(style)
+	err := saver.SaveAsYAML(file, outputPath, true)
+	if err != nil {
+ return err
+	}
+	return nil
+}
+
+func convertLinksToAbsoluteUrl(s string) string {
+	if s == "" {
+ return s
+	}
+	base := "https://docs.databricks.com"
+	referencePage := "/dev-tools/bundles/reference.html"
+
+	// Regular expression to match Markdown-style links like [_](link)
+	re := regexp.MustCompile(`\[_\]\(([^)]+)\)`)
+	result := re.ReplaceAllStringFunc(s, func(match string) string {
+ matches := re.FindStringSubmatch(match)
+ if len(matches) < 2 {
+ return match
+ }
+ link := matches[1]
+ var text, absoluteURL string
+
+ if strings.HasPrefix(link, "#") {
+ text = strings.TrimPrefix(link, "#")
+ absoluteURL = fmt.Sprintf("%s%s%s", base, referencePage, link)
+
+ // Handle relative paths like /dev-tools/bundles/resources.html#dashboard
+ } else if strings.HasPrefix(link, "/") {
+ absoluteURL = strings.ReplaceAll(fmt.Sprintf("%s%s", base, link), ".md", ".html")
+ if strings.Contains(link, "#") {
+ parts := strings.Split(link, "#")
+ text = parts[1]
+ } else {
+ text = "link"
+ }
+ } else {
+ return match
+ }
+
+ return fmt.Sprintf("[%s](%s)", text, absoluteURL)
+	})
+
+	return result
+}