Add support for cloning repositories from github

internal/git_clone_test.go

@@ -0,0 +1,63 @@
+package internal
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/cli/libs/git"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAccGitClonePublicRepository(t *testing.T) {
+	t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV"))
+
+	tmpDir := t.TempDir()
+	ctx := context.Background()
+	var err error
+
+	// We unset PATH to ensure that git.Clone cannot rely on the git CLI
+	t.Setenv("PATH", "")
+
+	err = git.Clone(ctx, git.CloneOptions{
+ Provider: "github",
+ Organization: "databricks",
+ RepositoryName: "cli",
+ Reference: "main",
+ TargetDir: tmpDir,
+	})
+
+	assert.NoError(t, err)
+	assert.DirExists(t, filepath.Join(tmpDir, "cli-main"))
+
+	b, err := os.ReadFile(filepath.Join(tmpDir, "cli-main/NOTICE"))
+	assert.NoError(t, err)
+	assert.Contains(t, string(b), "Copyright (2023) Databricks, Inc.")
+}
+
+func TestAccGitClonePublicRepositoryForTagReference(t *testing.T) {
+	t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV"))
+
+	tmpDir := t.TempDir()
+	ctx := context.Background()
+	var err error
+
+	// We unset PATH to ensure that git.Clone cannot rely on the git CLI
+	t.Setenv("PATH", "")
+
+	err = git.Clone(ctx, git.CloneOptions{
+ Provider: "github",
+ Organization: "databricks",
+ RepositoryName: "cli",
+ Reference: "snapshot",
+ TargetDir: tmpDir,
+	})
+
+	assert.NoError(t, err)
+	assert.DirExists(t, filepath.Join(tmpDir, "cli-snapshot"))
+
+	b, err := os.ReadFile(filepath.Join(tmpDir, "cli-snapshot/NOTICE"))
+	assert.NoError(t, err)
+	assert.Contains(t, string(b), "Copyright (2023) Databricks, Inc.")
+}

libs/cmdio/io.go

@@ -78,6 +78,21 @@ func IsErrTTY(ctx context.Context) bool {
 	return IsTTY(c.err)
 }
 
+func In(ctx context.Context) io.Reader {
+	c := fromContext(ctx)
+	return c.in
+}
+
+func Out(ctx context.Context) io.Writer {
+	c := fromContext(ctx)
+	return c.out
+}
+
+func Err(ctx context.Context) io.Writer {
+	c := fromContext(ctx)
+	return c.err
+}
+
 // IsTTY detects if stdout is a terminal. It assumes that stderr is terminal as well
 func (c *cmdIO) IsTTY() bool {
 	f, ok := c.out.(*os.File)

libs/git/clone.go

@@ -0,0 +1,147 @@
+package git
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/zip"
+)
+
+var errNotFound = errors.New("not found")
+
+type RepositoryNotFoundError struct {
+	url string
+}
+
+func (err RepositoryNotFoundError) Error() string {
+	return fmt.Sprintf("repository not found: %s", err.url)
+}
+
+func (err RepositoryNotFoundError) Is(other error) bool {
+	return other == errNotFound
+}
+
+type CloneOptions struct {
+	// Name of the organization or profile with the repository
+	Organization string
+	RepositoryName string
+
+	// Git service provider. Eg: github, gitlab
+	Provider string
+
+	// Branch or tag name to clone
+	Reference string
+
+	// Path to clone into. The repository is cloned as ${RepositoryName}-${Reference}
+	// in this target directory.
+	TargetDir string
+}
+
+func (opts CloneOptions) repoUrl() string {
+	return fmt.Sprintf(`https://github.com/%s/%s`, opts.Organization, opts.RepositoryName)
+}
+
+func (opts CloneOptions) zipUrl() string {
+	return fmt.Sprintf(`%s/archive/%s.zip`, opts.repoUrl(), opts.Reference)
+}
+
+func (opts CloneOptions) destination() string {
+	return filepath.Join(opts.TargetDir, opts.RepositoryName+"-"+opts.Reference)
+}
+
+func download(ctx context.Context, url string, dest string) error {
+	// Get request to download the ZIP archive
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+ return err
+	}
+	resp, err := http.DefaultClient.Do(request)
+	if err != nil {
+ return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusNotFound {
+ return RepositoryNotFoundError{url}
+	}
+
+	if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("failed to download ZIP archive: %s. %s", url, resp.Status)
+	}
+
+	f, err := os.Create(dest)
+	if err != nil {
+ return err
+	}
+	defer f.Close()
+
+	_, err = io.Copy(f, resp.Body)
+	return err
+}
+
+func clonePrivate(ctx context.Context, opts CloneOptions) error {
+	cmd := exec.CommandContext(ctx, "git", "clone", opts.repoUrl(), opts.destination(), "--branch", opts.Reference)
+
+	// Redirect exec command output
+	cmd.Stderr = cmdio.Err(ctx)
+	cmd.Stdout = cmdio.Out(ctx)
+	cmd.Stdin = cmdio.In(ctx)
+
+	// start git clone
+	err := cmd.Start()
+	if errors.Is(err, exec.ErrNotFound) {
+ return fmt.Errorf("please install git CLI to download private templates: %w", err)
+	}
+	if err != nil {
+ return err
+	}
+
+	// wait for git clone to complete
+	return cmd.Wait()
+}
+
+func clonePublic(ctx context.Context, opts CloneOptions) error {
+	tmpDir := os.TempDir()
+	defer os.Remove(tmpDir)
+
+	zipDst := filepath.Join(tmpDir, opts.RepositoryName+".zip")
+
+	// Download public repository from github as a ZIP file
+	err := download(ctx, opts.zipUrl(), zipDst)
+	if err != nil {
+ return err
+	}
+	defer os.Remove(zipDst)
+
+	// Decompress the ZIP file
+	err = zip.Extract(zipDst, opts.TargetDir)
+	if err != nil {
+ return err
+	}
+
+	// Remove the ZIP file post extraction
+	return os.Remove(zipDst)
+}
+
+func Clone(ctx context.Context, opts CloneOptions) error {
+	if opts.Provider != "github" {
+ return fmt.Errorf("git provider not supported: %s", opts.Provider)
+	}
+
+	// First we try to clone the repository as a public URL, as that does not
+	// require the git CLI
+	err := clonePublic(ctx, opts)
+
+	// If a public repository was not found, we defer to the git CLI
+	if errors.Is(err, errNotFound) {
+ return clonePrivate(ctx, opts)
+	}
+	return err
+}

libs/git/clone_test.go

@@ -0,0 +1,30 @@
+package git
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"testing"
+
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGitCloneCLINotFound(t *testing.T) {
+	// Set PATH to "", so git CLI cannot be found
+	t.Setenv("PATH", "")
+
+	tmpDir := t.TempDir()
+	cmdIO := cmdio.NewIO("text", os.Stdin, os.Stdout, os.Stderr, "")
+	ctx := cmdio.InContext(context.Background(), cmdIO)
+
+	err := Clone(ctx, CloneOptions{
+ Provider: "github",
+ Organization: "databricks",
+ RepositoryName: "does-not-exist",
+ Reference: "main",
+ TargetDir: tmpDir,
+	})
+	assert.ErrorIs(t, err, exec.ErrNotFound)
+	assert.ErrorContains(t, err, "please install git CLI to download private templates")
+}

libs/zip/extract.go

@@ -0,0 +1,49 @@
+package zip
+
+import (
+	"archive/zip"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+func Extract(src string, dst string) error {
+	zipReader, err := zip.OpenReader(src)
+	if err != nil {
+ return err
+	}
+	defer zipReader.Close()
+
+	// create dst directory incase it does not exist.
+	err = os.MkdirAll(dst, 0755)
+	if err != nil {
+ return err
+	}
+
+	return fs.WalkDir(zipReader, ".", func(path string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+
+ targetPath := filepath.Join(dst, path)
+ if d.IsDir() {
+ return os.MkdirAll(targetPath, 0755)
+ }
+
+ targetFile, err := os.Create(targetPath)
+ if err != nil {
+ return err
+ }
+ defer targetFile.Close()
+
+ sourceFile, err := zipReader.Open(path)
+ if err != nil {
+ return err
+ }
+ defer sourceFile.Close()
+
+ _, err = io.Copy(targetFile, sourceFile)
+ return err
+	})
+}

libs/zip/extract_test.go

@@ -0,0 +1,27 @@
+package zip
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestZipExtract(t *testing.T) {
+	tmpDir := t.TempDir()
+	var err error
+
+	err = Extract("./testdata/dir.zip", tmpDir)
+	assert.NoError(t, err)
+
+	assert.DirExists(t, filepath.Join(tmpDir, "dir"))
+
+	b, err := os.ReadFile(filepath.Join(tmpDir, "dir/a"))
+	assert.NoError(t, err)
+	assert.Equal(t, "hello a\n", string(b))
+
+	b, err = os.ReadFile(filepath.Join(tmpDir, "dir/b"))
+	assert.NoError(t, err)
+	assert.Equal(t, "hello b\n", string(b))
+}