Persist deployment metadata in WSFS

bundle/bundle.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/env"
+	"github.com/databricks/cli/bundle/metadata"
 	"github.com/databricks/cli/folders"
 	"github.com/databricks/cli/libs/git"
 	"github.com/databricks/cli/libs/locker"
@@ -31,6 +32,14 @@ const internalFolder = ".internal"
 type Bundle struct {
 	Config config.Root
 
+	// Metadata about the bundle deployment. This is the interface Databricks services
+	// rely on to integrate with bundles when they need additional information about
+	// a bundle deployment.
+	//
+	// After deploy, a file containing the metadata (metadata.json) can be found
+	// in the WSFS location containing the bundle state.
+	Metadata metadata.Metadata
+
 	// Store a pointer to the workspace client.
 	// It can be initialized on demand after loading the configuration.
 	clientOnce sync.Once

bundle/config/bundle.go

@@ -29,7 +29,7 @@ type Bundle struct {
 	Lock Lock `json:"lock" bundle:"readonly"`
 
 	// Force-override Git branch validation.
-	Force bool `json:"force" bundle:"readonly"`
+	Force bool `json:"force,omitempty" bundle:"readonly"`
 
 	// Contains Git information like current commit, current branch and
 	// origin url. Automatically loaded by reading .git directory if not specified

bundle/config/git.go

@@ -5,6 +5,9 @@ type Git struct {
 	OriginURL string `json:"origin_url,omitempty"`
 	Commit string `json:"commit,omitempty" bundle:"readonly"`
 
+	// Path to bundle root relative to the git repository root.
+	BundleRootPath string `json:"bundle_root_path,omitempty" bundle:"readonly"`
+
 	// Inferred is set to true if the Git details were inferred and weren't set explicitly
 	Inferred bool `json:"-" bundle:"readonly"`
 

bundle/config/lock.go

@@ -4,11 +4,11 @@ type Lock struct {
 	// Enabled toggles deployment lock. True by default.
 	// Use a pointer value so that only explicitly configured values are set
 	// and we don't merge configuration with zero-initialized values.
-	Enabled *bool `json:"enabled"`
+	Enabled *bool `json:"enabled,omitempty"`
 
 	// Force acquisition of deployment lock even if it is currently held.
 	// This may be necessary if a prior deployment failed to release the lock.
-	Force bool `json:"force"`
+	Force bool `json:"force,omitempty"`
 }
 
 func (lock Lock) IsEnabled() bool {

bundle/config/mutator/load_git_details.go

@@ -2,6 +2,7 @@ package mutator
 
 import (
 	"context"
+	"path/filepath"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/libs/git"
@@ -52,5 +53,17 @@ func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) error {
  remoteUrl := repo.OriginUrl()
  b.Config.Bundle.Git.OriginURL = remoteUrl
 	}
+
+	// Compute relative path of the bundle root from the Git repo root.
+	absBundlePath, err := filepath.Abs(b.Config.Path)
+	if err != nil {
+ return err
+	}
+	// repo.Root() returns the absolute path of the repo
+	relBundlePath, err := filepath.Rel(repo.Root(), absBundlePath)
+	if err != nil {
+ return err
+	}
+	b.Config.Bundle.Git.BundleRootPath = filepath.ToSlash(relBundlePath)
 	return nil
 }

bundle/config/paths/paths.go

@@ -6,8 +6,8 @@ import (
 )
 
 type Paths struct {
-	// ConfigFilePath holds the path to the configuration file that
-	// described the resource that this type is embedded in.
+	// Absolute path on the local file system to the configuration file that holds
+	// the definition of this resource.
 	ConfigFilePath string `json:"-" bundle:"readonly"`
 }
 

bundle/deploy/metadata/compute.go

@@ -0,0 +1,51 @@
+package metadata
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/metadata"
+)
+
+type compute struct{}
+
+func Compute() bundle.Mutator {
+	return &compute{}
+}
+
+func (m *compute) Name() string {
+	return "metadata.Compute"
+}
+
+func (m *compute) Apply(_ context.Context, b *bundle.Bundle) error {
+	b.Metadata = metadata.Metadata{
+ Version: metadata.Version,
+ Config: metadata.Config{},
+	}
+
+	// Set git details in metadata
+	b.Metadata.Config.Bundle.Git = b.Config.Bundle.Git
+
+	// Set job config paths in metadata
+	jobsMetadata := make(map[string]*metadata.Job)
+	for name, job := range b.Config.Resources.Jobs {
+ // Compute config file path the job is defined in, relative to the bundle
+ // root
+ relativePath, err := filepath.Rel(b.Config.Path, job.ConfigFilePath)
+ if err != nil {
+ return fmt.Errorf("failed to compute relative path for job %s: %w", name, err)
+ }
+ // Metadata for the job
+ jobsMetadata[name] = &metadata.Job{
+ ID: job.ID,
+ RelativePath: filepath.ToSlash(relativePath),
+ }
+	}
+	b.Metadata.Config.Resources.Jobs = jobsMetadata
+
+	// Set file upload destination of the bundle in metadata
+	b.Metadata.Config.Workspace.FilesPath = b.Config.Workspace.FilesPath
+	return nil
+}

bundle/deploy/metadata/compute_test.go

@@ -0,0 +1,100 @@
+package metadata
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/paths"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/metadata"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestComputeMetadataMutator(t *testing.T) {
+	b := &bundle.Bundle{
+ Config: config.Root{
+ Workspace: config.Workspace{
+ RootPath: "/Users/shreyas.goenka@databricks.com",
+ ArtifactsPath: "/Users/shreyas.goenka@databricks.com/artifacts",
+ FilesPath: "/Users/shreyas.goenka@databricks.com/files",
+ },
+ Bundle: config.Bundle{
+ Name: "my-bundle",
+ Target: "development",
+ Git: config.Git{
+ Branch: "my-branch",
+ OriginURL: "www.host.com",
+ Commit: "abcd",
+ BundleRootPath: "a/b/c/d",
+ },
+ },
+ Resources: config.Resources{
+ Jobs: map[string]*resources.Job{
+ "my-job-1": {
+ Paths: paths.Paths{
+ ConfigFilePath: "a/b/c",
+ },
+ ID: "1111",
+ JobSettings: &jobs.JobSettings{
+ Name: "My Job One",
+ },
+ },
+ "my-job-2": {
+ Paths: paths.Paths{
+ ConfigFilePath: "d/e/f",
+ },
+ ID: "2222",
+ JobSettings: &jobs.JobSettings{
+ Name: "My Job Two",
+ },
+ },
+ },
+ Pipelines: map[string]*resources.Pipeline{
+ "my-pipeline": {
+ Paths: paths.Paths{
+ ConfigFilePath: "abc",
+ },
+ },
+ },
+ },
+ },
+	}
+
+	expectedMetadata := metadata.Metadata{
+ Version: metadata.Version,
+ Config: metadata.Config{
+ Workspace: metadata.Workspace{
+ FilesPath: "/Users/shreyas.goenka@databricks.com/files",
+ },
+ Bundle: metadata.Bundle{
+ Git: config.Git{
+ Branch: "my-branch",
+ OriginURL: "www.host.com",
+ Commit: "abcd",
+ BundleRootPath: "a/b/c/d",
+ },
+ },
+ Resources: metadata.Resources{
+ Jobs: map[string]*metadata.Job{
+ "my-job-1": {
+ RelativePath: "a/b/c",
+ ID: "1111",
+ },
+ "my-job-2": {
+ RelativePath: "d/e/f",
+ ID: "2222",
+ },
+ },
+ },
+ },
+	}
+
+	err := Compute().Apply(context.Background(), b)
+	require.NoError(t, err)
+
+	assert.Equal(t, expectedMetadata, b.Metadata)
+}

bundle/deploy/metadata/upload.go

@@ -0,0 +1,36 @@
+package metadata
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/filer"
+)
+
+const MetadataFileName = "metadata.json"
+
+type upload struct{}
+
+func Upload() bundle.Mutator {
+	return &upload{}
+}
+
+func (m *upload) Name() string {
+	return "metadata.Upload"
+}
+
+func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) error {
+	f, err := filer.NewWorkspaceFilesClient(b.WorkspaceClient(), b.Config.Workspace.StatePath)
+	if err != nil {
+ return err
+	}
+
+	metadata, err := json.MarshalIndent(b.Metadata, "", " ")
+	if err != nil {
+ return err
+	}
+
+	return f.Write(ctx, MetadataFileName, bytes.NewReader(metadata), filer.CreateParentDirectories, filer.OverwriteIfExists)
+}

bundle/metadata/metadata.go

@@ -0,0 +1,45 @@
+package metadata
+
+import (
+	"github.com/databricks/cli/bundle/config"
+)
+
+const Version = 1
+
+type Bundle struct {
+	Git config.Git `json:"git,omitempty"`
+}
+
+type Workspace struct {
+	FilesPath string `json:"file_path,omitempty"`
+}
+
+type Job struct {
+	ID string `json:"id,omitempty"`
+
+	// Relative path from the bundle root to the configuration file that holds
+	// the definition of this resource.
+	RelativePath string `json:"relative_path,omitempty"`
+}
+
+type Resources struct {
+	Jobs map[string]*Job `json:"jobs,omitempty"`
+}
+
+type Config struct {
+	Bundle Bundle `json:"bundle,omitempty"`
+	Workspace Workspace `json:"workspace,omitempty"`
+	Resources Resources `json:"resources,omitempty"`
+}
+
+// Metadata about the bundle deployment. This is the interface Databricks services
+// rely on to integrate with bundles when they need additional information about
+// a bundle deployment.
+//
+// After deploy, a file containing the metadata (metadata.json) can be found
+// in the WSFS location containing the bundle state.
+type Metadata struct {
+	Version int `json:"version"`
+
+	Config Config `json:"config"`
+}

bundle/phases/deploy.go

@@ -7,6 +7,7 @@ import (
 	"github.com/databricks/cli/bundle/config/mutator"
 	"github.com/databricks/cli/bundle/deploy/files"
 	"github.com/databricks/cli/bundle/deploy/lock"
+	"github.com/databricks/cli/bundle/deploy/metadata"
 	"github.com/databricks/cli/bundle/deploy/terraform"
 	"github.com/databricks/cli/bundle/libraries"
 	"github.com/databricks/cli/bundle/python"
@@ -31,7 +32,12 @@ func Deploy() bundle.Mutator {
  terraform.StatePull(),
  bundle.Defer(
  terraform.Apply(),
- terraform.StatePush(),
+ bundle.Seq(
+ terraform.StatePush(),
+ terraform.Load(),
+ metadata.Compute(),
+ metadata.Upload(),
+ ),
  ),
  ),
  lock.Release(lock.GoalDeploy),

internal/bundle/bundles/deploy_then_remove_resources/databricks_template_schema.json

@@ -2,7 +2,7 @@
  "properties": {
  "unique_id": {
  "type": "string",
- "description": "Unique ID for job name"
+ "description": "Unique ID for pipeline name"
  }
  }
 }

internal/bundle/bundles/job_metadata/databricks_template_schema.json

@@ -0,0 +1,16 @@
+{
+ "properties": {
+ "unique_id": {
+ "type": "string",
+ "description": "Unique ID for job name"
+ },
+ "spark_version": {
+ "type": "string",
+ "description": "Spark version used for job cluster"
+ },
+ "node_type_id": {
+ "type": "string",
+ "description": "Node type id for job cluster"
+ }
+ }
+}

internal/bundle/bundles/job_metadata/template/a/b/bar.py

@@ -0,0 +1,2 @@
+# Databricks notebook source
+print("bye")

internal/bundle/bundles/job_metadata/template/a/b/resources.yml.tmpl

@@ -0,0 +1,12 @@
+resources:
+ jobs:
+ bar:
+ name: test-job-metadata-2-{{.unique_id}}
+ tasks:
+ - task_key: my_notebook_task
+ new_cluster:
+ num_workers: 1
+ spark_version: "{{.spark_version}}"
+ node_type_id: "{{.node_type_id}}"
+ notebook_task:
+ notebook_path: "./bar.py"