Add databricks labs commad group

..

Add folder

list installed works

stdin forwarding

fixed some tests

tests pass

fetching versions

added installer, still WIP

fixed tests for UNIX

added debug logging

..

installer tests

..

run on windows

xx

fixed

added uninstall

..

work on windows

..

..

work with more CUJs

pick up configuration when auth.json is not found

..

Added spinners

added cluster prompts

..

...

..

support for dbconnect lib install

..

..

..

fixed dev libDIr logic

starting tests for dev installs

+10% coverage

,,

..

..

..

Author: nfx

cmd/cmd.go

@@ -10,6 +10,7 @@ import (
 	"github.com/databricks/cli/cmd/bundle"
 	"github.com/databricks/cli/cmd/configure"
 	"github.com/databricks/cli/cmd/fs"
+	"github.com/databricks/cli/cmd/labs"
 	"github.com/databricks/cli/cmd/root"
 	"github.com/databricks/cli/cmd/sync"
 	"github.com/databricks/cli/cmd/version"
@@ -70,6 +71,7 @@ func New(ctx context.Context) *cobra.Command {
 	cli.AddCommand(bundle.New())
 	cli.AddCommand(configure.New())
 	cli.AddCommand(fs.New())
+	cli.AddCommand(labs.New(ctx))
 	cli.AddCommand(sync.New())
 	cli.AddCommand(version.New())
 

cmd/labs/CODEOWNERS

@@ -0,0 +1 @@
+* @nfx

cmd/labs/clear_cache.go

@@ -0,0 +1,32 @@
+package labs
+
+import (
+	"log/slog"
+	"os"
+
+	"github.com/databricks/cli/cmd/labs/project"
+	"github.com/databricks/cli/libs/log"
+	"github.com/spf13/cobra"
+)
+
+func newClearCacheCommand() *cobra.Command {
+	return &cobra.Command{
+		Use:   "clear-cache",
+		Short: "Clears cache entries from everywhere relevant",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx := cmd.Context()
+			projects, err := project.Installed(ctx)
+			if err != nil {
+				return err
+			}
+			_ = os.Remove(project.PathInLabs(ctx, "repositories.json"))
+			logger := log.GetLogger(ctx)
+			for _, prj := range projects {
+				logger.Info("clearing labs project cache", slog.String("name", prj.Name))
+				_ = os.RemoveAll(prj.CacheDir(ctx))
+				_ = prj.EnsureFoldersExist(ctx)
+			}
+			return nil
+		},
+	}
+}

cmd/labs/github/github.go

@@ -0,0 +1,66 @@
+package github
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/databricks/cli/libs/log"
+)
+
+const gitHubAPI = "https://api.github.com"
+const gitHubUserContent = "https://raw.githubusercontent.com"
+
+// Placeholders to use as unique keys in context.Context.
+var apiOverride int
+var userContentOverride int
+
+func WithApiOverride(ctx context.Context, override string) context.Context {
+	return context.WithValue(ctx, &apiOverride, override)
+}
+
+func WithUserContentOverride(ctx context.Context, override string) context.Context {
+	return context.WithValue(ctx, &userContentOverride, override)
+}
+
+var ErrNotFound = errors.New("not found")
+
+func getBytes(ctx context.Context, method, url string, body io.Reader) ([]byte, error) {
+	ao, ok := ctx.Value(&apiOverride).(string)
+	if ok {
+		url = strings.Replace(url, gitHubAPI, ao, 1)
+	}
+	uco, ok := ctx.Value(&userContentOverride).(string)
+	if ok {
+		url = strings.Replace(url, gitHubUserContent, uco, 1)
+	}
+	log.Tracef(ctx, "%s %s", method, url)
+	req, err := http.NewRequestWithContext(ctx, "GET", url, body)
+	if err != nil {
+		return nil, err
+	}
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if res.StatusCode == 404 {
+		return nil, ErrNotFound
+	}
+	if res.StatusCode >= 400 {
+		return nil, fmt.Errorf("github request failed: %s", res.Status)
+	}
+	defer res.Body.Close()
+	return io.ReadAll(res.Body)
+}
+
+func httpGetAndUnmarshall(ctx context.Context, url string, response any) error {
+	raw, err := getBytes(ctx, "GET", url, nil)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(raw, response)
+}

cmd/labs/github/ref.go

@@ -0,0 +1,20 @@
+package github
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/cli/libs/log"
+)
+
+func ReadFileFromRef(ctx context.Context, org, repo, ref, file string) ([]byte, error) {
+	log.Debugf(ctx, "Reading %s@%s from %s/%s", file, ref, org, repo)
+	url := fmt.Sprintf("%s/%s/%s/%s/%s", gitHubUserContent, org, repo, ref, file)
+	return getBytes(ctx, "GET", url, nil)
+}
+
+func DownloadZipball(ctx context.Context, org, repo, ref string) ([]byte, error) {
+	log.Debugf(ctx, "Downloading zipball for %s from %s/%s", ref, org, repo)
+	zipballURL := fmt.Sprintf("%s/repos/%s/%s/zipball/%s", gitHubAPI, org, repo, ref)
+	return getBytes(ctx, "GET", zipballURL, nil)
+}

cmd/labs/github/ref_test.go

@@ -0,0 +1,48 @@
+package github
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFileFromRef(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/databrickslabs/ucx/main/README.md" {
+			w.Write([]byte(`abc`))
+			return
+		}
+		t.Logf("Requested: %s", r.URL.Path)
+		panic("stub required")
+	}))
+	defer server.Close()
+
+	ctx := context.Background()
+	ctx = WithUserContentOverride(ctx, server.URL)
+
+	raw, err := ReadFileFromRef(ctx, "databrickslabs", "ucx", "main", "README.md")
+	assert.NoError(t, err)
+	assert.Equal(t, []byte("abc"), raw)
+}
+
+func TestDownloadZipball(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/repos/databrickslabs/ucx/zipball/main" {
+			w.Write([]byte(`abc`))
+			return
+		}
+		t.Logf("Requested: %s", r.URL.Path)
+		panic("stub required")
+	}))
+	defer server.Close()
+
+	ctx := context.Background()
+	ctx = WithApiOverride(ctx, server.URL)
+
+	raw, err := DownloadZipball(ctx, "databrickslabs", "ucx", "main")
+	assert.NoError(t, err)
+	assert.Equal(t, []byte("abc"), raw)
+}

cmd/labs/github/releases.go

@@ -0,0 +1,61 @@
+package github
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/databricks/cli/cmd/labs/localcache"
+	"github.com/databricks/cli/libs/log"
+)
+
+const cacheTTL = 1 * time.Hour
+
+// NewReleaseCache creates a release cache for a repository in the GitHub org.
+// Caller has to provide different cache directories for different repositories.
+func NewReleaseCache(org, repo, cacheDir string) *ReleaseCache {
+	pattern := fmt.Sprintf("%s-%s-releases", org, repo)
+	return &ReleaseCache{
+		cache: localcache.NewLocalCache[Versions](cacheDir, pattern, cacheTTL),
+		Org:   org,
+		Repo:  repo,
+	}
+}
+
+type ReleaseCache struct {
+	cache localcache.LocalCache[Versions]
+	Org   string
+	Repo  string
+}
+
+func (r *ReleaseCache) Load(ctx context.Context) (Versions, error) {
+	return r.cache.Load(ctx, func() (Versions, error) {
+		return getVersions(ctx, r.Org, r.Repo)
+	})
+}
+
+// getVersions is considered to be a private API, as we want the usage go through a cache
+func getVersions(ctx context.Context, org, repo string) (Versions, error) {
+	var releases Versions
+	log.Debugf(ctx, "Fetching latest releases for %s/%s from GitHub API", org, repo)
+	url := fmt.Sprintf("%s/repos/%s/%s/releases", gitHubAPI, org, repo)
+	err := httpGetAndUnmarshall(ctx, url, &releases)
+	return releases, err
+}
+
+type ghAsset struct {
+	Name               string `json:"name"`
+	ContentType        string `json:"content_type"`
+	Size               int    `json:"size"`
+	BrowserDownloadURL string `json:"browser_download_url"`
+}
+
+type Release struct {
+	Version     string    `json:"tag_name"`
+	CreatedAt   time.Time `json:"created_at"`
+	PublishedAt time.Time `json:"published_at"`
+	ZipballURL  string    `json:"zipball_url"`
+	Assets      []ghAsset `json:"assets"`
+}
+
+type Versions []Release

cmd/labs/github/releases_test.go

@@ -0,0 +1,34 @@
+package github
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadsReleasesForCLI(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/repos/databricks/cli/releases" {
+			w.Write([]byte("[]"))
+			return
+		}
+		t.Logf("Requested: %s", r.URL.Path)
+		panic("stub required")
+	}))
+	defer server.Close()
+
+	ctx := context.Background()
+	ctx = WithApiOverride(ctx, server.URL)
+
+	r := NewReleaseCache("databricks", "cli", t.TempDir())
+	all, err := r.Load(ctx)
+	assert.NoError(t, err)
+	assert.NotNil(t, all)
+
+	// no call is made
+	_, err = r.Load(ctx)
+	assert.NoError(t, err)
+}

cmd/labs/github/repositories.go

@@ -0,0 +1,59 @@
+package github
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/databricks/cli/cmd/labs/localcache"
+	"github.com/databricks/cli/libs/log"
+)
+
+const repositoryCacheTTL = 24 * time.Hour
+
+func NewRepositoryCache(org, cacheDir string) *repositoryCache {
+	filename := fmt.Sprintf("%s-repositories", org)
+	return &repositoryCache{
+		cache: localcache.NewLocalCache[Repositories](cacheDir, filename, repositoryCacheTTL),
+		Org:   org,
+	}
+}
+
+type repositoryCache struct {
+	cache localcache.LocalCache[Repositories]
+	Org   string
+}
+
+func (r *repositoryCache) Load(ctx context.Context) (Repositories, error) {
+	return r.cache.Load(ctx, func() (Repositories, error) {
+		return getRepositories(ctx, r.Org)
+	})
+}
+
+// getRepositories is considered to be privata API, as we want the usage to go through a cache
+func getRepositories(ctx context.Context, org string) (Repositories, error) {
+	var repos Repositories
+	log.Debugf(ctx, "Loading repositories for %s from GitHub API", org)
+	url := fmt.Sprintf("%s/users/%s/repos", gitHubAPI, org)
+	err := httpGetAndUnmarshall(ctx, url, &repos)
+	return repos, err
+}
+
+type Repositories []ghRepo
+
+type ghRepo struct {
+	Name          string   `json:"name"`
+	Description   string   `json:"description"`
+	Langauge      string   `json:"language"`
+	DefaultBranch string   `json:"default_branch"`
+	Stars         int      `json:"stargazers_count"`
+	IsFork        bool     `json:"fork"`
+	IsArchived    bool     `json:"archived"`
+	Topics        []string `json:"topics"`
+	HtmlURL       string   `json:"html_url"`
+	CloneURL      string   `json:"clone_url"`
+	SshURL        string   `json:"ssh_url"`
+	License       struct {
+		Name string `json:"name"`
+	} `json:"license"`
+}

cmd/labs/github/repositories_test.go

@@ -0,0 +1,30 @@
+package github
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRepositories(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/users/databrickslabs/repos" {
+			w.Write([]byte(`[{"name": "x"}]`))
+			return
+		}
+		t.Logf("Requested: %s", r.URL.Path)
+		panic("stub required")
+	}))
+	defer server.Close()
+
+	ctx := context.Background()
+	ctx = WithApiOverride(ctx, server.URL)
+
+	r := NewRepositoryCache("databrickslabs", t.TempDir())
+	all, err := r.Load(ctx)
+	assert.NoError(t, err)
+	assert.True(t, len(all) > 0)
+}