Make bundle JSON schema modular with $defs

.codegen.json

@@ -11,10 +11,10 @@
  "toolchain": {
  "required": ["go"],
  "post_generate": [
- "go run ./bundle/internal/bundle/schema/main.go ./bundle/schema/docs/bundle_descriptions.json",
+ "go run ./bundle/internal/schema/*.go ./bundle/schema/jsonschema.json",
  "echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes",
  "echo 'go.sum linguist-generated=true' >> ./.gitattributes",
- "echo 'bundle/schema/docs/bundle_descriptions.json linguist-generated=true' >> ./.gitattributes"
+ "echo 'bundle/schema/jsonschema.json linguist-generated=true' >> ./.gitattributes"
  ]
  }
 }

.gitattributes

@@ -120,4 +120,4 @@ cmd/workspace/workspace-conf/workspace-conf.go linguist-generated=true
 cmd/workspace/workspace/workspace.go linguist-generated=true
 bundle/internal/tf/schema/\*.go linguist-generated=true
 go.sum linguist-generated=true
-bundle/schema/docs/bundle_descriptions.json linguist-generated=true
+bundle/schema/jsonschema.json linguist-generated=true

bundle/internal/schema/main.go

@@ -0,0 +1,93 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"reflect"
+
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/variable"
+	"github.com/databricks/cli/libs/jsonschema"
+)
+
+func interpolationPattern(s string) string {
+	return fmt.Sprintf(`\$\{(%s(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)+)\}`, s)
+}
+
+func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
+	if typ == reflect.TypeOf(config.Root{}) || typ == reflect.TypeOf(variable.Variable{}) {
+ return s
+	}
+
+	switch s.Type {
+	case jsonschema.ArrayType, jsonschema.ObjectType:
+ // arrays and objects can have complex variable values specified.
+ return jsonschema.Schema{
+ AnyOf: []jsonschema.Schema{
+ s,
+ {
+ Type: jsonschema.StringType,
+ Pattern: interpolationPattern("var"),
+ }},
+ }
+	case jsonschema.IntegerType, jsonschema.NumberType, jsonschema.BooleanType:
+ // primitives can have variable values, or references like ${bundle.xyz}
+ // or ${workspace.xyz}
+ return jsonschema.Schema{
+ AnyOf: []jsonschema.Schema{
+ s,
+ {Type: jsonschema.StringType, Pattern: interpolationPattern("resources")},
+ {Type: jsonschema.StringType, Pattern: interpolationPattern("bundle")},
+ {Type: jsonschema.StringType, Pattern: interpolationPattern("workspace")},
+ {Type: jsonschema.StringType, Pattern: interpolationPattern("artifacts")},
+ {Type: jsonschema.StringType, Pattern: interpolationPattern("var")},
+ },
+ }
+	default:
+ return s
+	}
+}
+
+func main() {
+	if len(os.Args) != 2 {
+ fmt.Println("Usage: go run main.go <output-file>")
+ os.Exit(1)
+	}
+
+	// Output file, where the generated JSON schema will be written to.
+	outputFile := os.Args[1]
+
+	// Input file, the databricks openapi spec.
+	inputFile := os.Getenv("DATABRICKS_OPENAPI_SPEC")
+	if inputFile == "" {
+ log.Fatal("DATABRICKS_OPENAPI_SPEC environment variable not set")
+	}
+
+	p, err := newParser(inputFile)
+	if err != nil {
+ log.Fatal(err)
+	}
+
+	// Generate the JSON schema from the bundle Go struct.
+	s, err := jsonschema.FromType(reflect.TypeOf(config.Root{}), []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{
+ p.addDescriptions,
+ p.addEnums,
+ addInterpolationPatterns,
+	})
+	if err != nil {
+ log.Fatal(err)
+	}
+
+	b, err := json.MarshalIndent(s, "", " ")
+	if err != nil {
+ log.Fatal(err)
+	}
+
+	// Write the schema descriptions to the output file.
+	err = os.WriteFile(outputFile, b, 0644)
+	if err != nil {
+ log.Fatal(err)
+	}
+}

bundle/internal/schema/parser.go

@@ -0,0 +1,123 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
+	"reflect"
+	"strings"
+
+	"github.com/databricks/cli/libs/jsonschema"
+)
+
+type Components struct {
+	Schemas map[string]jsonschema.Schema `json:"schemas,omitempty"`
+}
+
+type Specification struct {
+	Components Components `json:"components"`
+}
+
+type openapiParser struct {
+	ref map[string]jsonschema.Schema
+}
+
+func newParser(path string) (*openapiParser, error) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+ return nil, err
+	}
+
+	spec := Specification{}
+	err = json.Unmarshal(b, &spec)
+	if err != nil {
+ return nil, err
+	}
+
+	p := &openapiParser{}
+	p.ref = spec.Components.Schemas
+	return p, nil
+}
+
+// This function checks if the input type:
+// 1. Is a Databricks Go SDK type.
+// 2. Has a Databricks Go SDK type embedded in it.
+//
+// If the above conditions are met, the function returns the JSON schema
+// corresponding to the Databricks Go SDK type from the OpenAPI spec.
+func (p *openapiParser) findRef(typ reflect.Type) (jsonschema.Schema, bool) {
+	typs := []reflect.Type{typ}
+
+	// Check for embedded Databricks Go SDK types.
+	if typ.Kind() == reflect.Struct {
+ for i := 0; i < typ.NumField(); i++ {
+ if !typ.Field(i).Anonymous {
+ continue
+ }
+
+ // Deference current type if it's a pointer.
+ ctyp := typ.Field(i).Type
+ for ctyp.Kind() == reflect.Ptr {
+ ctyp = ctyp.Elem()
+ }
+
+ typs = append(typs, ctyp)
+ }
+	}
+
+	for _, ctyp := range typs {
+ // Skip if it's not a Go SDK type.
+ if !strings.HasPrefix(ctyp.PkgPath(), "github.com/databricks/databricks-sdk-go") {
+ continue
+ }
+
+ pkgName := path.Base(ctyp.PkgPath())
+ k := fmt.Sprintf("%s.%s", pkgName, ctyp.Name())
+
+ // Skip if the type is not in the openapi spec.
+ _, ok := p.ref[k]
+ if !ok {
+ continue
+ }
+
+ // Return the first Go SDK type found in the openapi spec.
+ return p.ref[k], true
+	}
+
+	return jsonschema.Schema{}, false
+}
+
+// Use the OpenAPI spec to load descriptions for the given type.
+func (p *openapiParser) addDescriptions(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
+	ref, ok := p.findRef(typ)
+	if !ok {
+ return s
+	}
+
+	s.Description = ref.Description
+	for k, v := range s.Properties {
+ if refProp, ok := ref.Properties[k]; ok {
+ v.Description = refProp.Description
+ }
+	}
+
+	return s
+}
+
+// Use the OpenAPI spec add enum values for the given type.
+func (p *openapiParser) addEnums(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
+	ref, ok := p.findRef(typ)
+	if !ok {
+ return s
+	}
+
+	s.Enum = append(s.Enum, ref.Enum...)
+	for k, v := range s.Properties {
+ if refProp, ok := ref.Properties[k]; ok {
+ v.Enum = append(v.Enum, refProp.Enum...)
+ }
+	}
+
+	return s
+}