Add archive build for DBR testing (#3452)

## Changes
This PR adds an internal program that can build a CLI test archive that
contains the entire repo and binaries like uv, go, and jq to run
acceptance tests.

This PR builds on top of #3444

## Why
A single deployable binary makes testing on DBR very straightforward.
All you then need to do is decompile the binary and run tests using a
test runner of your choice.

For now, the tests will be run using the go binary, but this could be
extended to run them using gotestsum.

## Tests
A new unit test. This has a build tag so that we can limit its execution
to only a DBR CI environment.

Author: shreyas-goenka

internal/testarchive/.gitignore

@@ -0,0 +1,2 @@
+_build/
+_bin/

internal/testarchive/archive.go

@@ -0,0 +1,183 @@
+package main
+
+import (
+	"archive/tar"
+	"bufio"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+// gitFiles returns a list of all git-tracked files in the repository.
+func gitFiles(repoRoot string) ([]string, error) {
+	cmd := exec.Command("git", "ls-files")
+	cmd.Dir = repoRoot
+	output, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get git tracked files: %w", err)
+	}
+
+	scanner := bufio.NewScanner(strings.NewReader(string(output)))
+	var gitFiles []string
+	for scanner.Scan() {
+		file := strings.TrimSpace(scanner.Text())
+		if file != "" {
+			gitFiles = append(gitFiles, file)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("error reading git ls-files output: %w", err)
+	}
+
+	return gitFiles, nil
+}
+
+func binFiles(binDir string) ([]string, error) {
+	var binFiles []string
+
+	err := filepath.WalkDir(binDir, func(path string, d os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if d.IsDir() {
+			return nil
+		}
+
+		relPath, err := filepath.Rel(binDir, path)
+		if err != nil {
+			return fmt.Errorf("failed to get relative path: %w", err)
+		}
+
+		binFiles = append(binFiles, relPath)
+		return nil
+	})
+
+	return binFiles, err
+}
+
+// addFileToArchive adds a single file to the tar archive
+func addFileToArchive(tarWriter *tar.Writer, src, dst string) error {
+	info, err := os.Stat(src)
+	if err != nil {
+		return fmt.Errorf("failed to stat: %w", err)
+	}
+
+	// Skip directories and non-regular files
+	if !info.Mode().IsRegular() {
+		return nil
+	}
+
+	header := &tar.Header{
+		Name:    dst,
+		Size:    info.Size(),
+		Mode:    int64(info.Mode()),
+		ModTime: info.ModTime(),
+	}
+	if err := tarWriter.WriteHeader(header); err != nil {
+		return fmt.Errorf("failed to write header: %w", err)
+	}
+
+	fileReader, err := os.Open(src)
+	if err != nil {
+		return fmt.Errorf("failed to open file: %w", err)
+	}
+	defer fileReader.Close()
+
+	_, err = io.Copy(tarWriter, fileReader)
+	if err != nil {
+		return fmt.Errorf("failed to copy file content: %w", err)
+	}
+
+	return nil
+}
+
+// createArchive creates a tar.gz archive of all git-tracked files plus downloaded tools
+func createArchive(archiveDir, binDir, repoRoot string) error {
+	archivePath := filepath.Join(archiveDir, "archive.tar.gz")
+
+	// Download tools for both arm and amd64 architectures.
+	// The right architecture to use is decided at runtime on the serverless driver.
+	// The Databricks platform explicitly does not provide any guarantees around
+	// the CPU architecture to keep the door open for future optimizations.
+	downloaders := []downloader{
+		goDownloader{arch: "amd64", binDir: binDir},
+		goDownloader{arch: "arm64", binDir: binDir},
+		uvDownloader{arch: "amd64", binDir: binDir},
+		uvDownloader{arch: "arm64", binDir: binDir},
+		jqDownloader{arch: "amd64", binDir: binDir},
+		jqDownloader{arch: "arm64", binDir: binDir},
+	}
+
+	for _, downloader := range downloaders {
+		err := downloader.Download()
+		if err != nil {
+			return fmt.Errorf("failed to download %s: %w", downloader, err)
+		}
+	}
+
+	gitFiles, err := gitFiles(repoRoot)
+	if err != nil {
+		return fmt.Errorf("failed to get git tracked files: %w", err)
+	}
+
+	binFiles, err := binFiles(binDir)
+	if err != nil {
+		return fmt.Errorf("failed to get bin files: %w", err)
+	}
+
+	totalFiles := len(gitFiles) + len(binFiles)
+	fmt.Printf("Found %d git-tracked files and %d downloaded files (%d total)\n",
+		len(gitFiles), len(binFiles), totalFiles)
+
+	// Create archive directory if it doesn't exist
+	if err := os.MkdirAll(archiveDir, 0o755); err != nil {
+		return fmt.Errorf("failed to create output directory: %w", err)
+	}
+
+	// Create the tar.gz file
+	outFile, err := os.Create(archivePath)
+	if err != nil {
+		return fmt.Errorf("failed to create output file: %w", err)
+	}
+	defer outFile.Close()
+
+	// Create gzip writer
+	gzWriter := gzip.NewWriter(outFile)
+	defer gzWriter.Close()
+
+	// Create tar.gz writer
+	tarWriter := tar.NewWriter(gzWriter)
+	defer tarWriter.Close()
+
+	fmt.Printf("Creating archive %s...\n", archivePath)
+
+	// Add git-tracked files to the archive
+	for _, file := range gitFiles {
+		err := addFileToArchive(tarWriter, filepath.Join(repoRoot, file), filepath.Join("cli", file))
+		if err != nil {
+			fmt.Printf("Warning: failed to add git file %s: %v\n", file, err)
+		}
+	}
+
+	// Add downloaded files / binaries to the archive
+	for _, file := range binFiles {
+		err := addFileToArchive(tarWriter, filepath.Join(binDir, file), filepath.Join("bin", file))
+		if err != nil {
+			fmt.Printf("Warning: failed to add downloaded file %s: %v\n", file, err)
+		}
+	}
+
+	stat, err := outFile.Stat()
+	if err != nil {
+		return fmt.Errorf("failed to stat archive: %w", err)
+	}
+
+	fmt.Printf("✅ Successfully created comprehensive archive. Archive size: %.1f MB\n", float64(stat.Size())/(1024*1024))
+	return nil
+}

internal/testarchive/archive_test.go

@@ -0,0 +1,36 @@
+//go:build dbr_only
+
+package main
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestArchive(t *testing.T) {
+	archiveDir := t.TempDir()
+	binDir := t.TempDir()
+	repoRoot := "../.."
+
+	err := createArchive(archiveDir, binDir, repoRoot)
+	require.NoError(t, err)
+
+	assertDir := t.TempDir()
+	err = extractTarGz(filepath.Join(archiveDir, "archive.tar.gz"), assertDir)
+	require.NoError(t, err)
+
+	// Go installation is a directory because it includes the
+	// standard library source code along with the Go binary.
+	assert.FileExists(t, filepath.Join(assertDir, "bin", "arm64", "go", "bin", "go"))
+	assert.FileExists(t, filepath.Join(assertDir, "bin", "amd64", "go", "bin", "go"))
+	assert.FileExists(t, filepath.Join(assertDir, "bin", "arm64", "uv"))
+	assert.FileExists(t, filepath.Join(assertDir, "bin", "amd64", "uv"))
+	assert.FileExists(t, filepath.Join(assertDir, "bin", "arm64", "jq"))
+	assert.FileExists(t, filepath.Join(assertDir, "bin", "amd64", "jq"))
+
+	assert.FileExists(t, filepath.Join(assertDir, "cli", "go.mod"))
+	assert.FileExists(t, filepath.Join(assertDir, "cli", "go.sum"))
+}

internal/testarchive/go.go

@@ -18,6 +18,10 @@ var (
 	_ = jqDownloader{}
 )
 
+type downloader interface {
+	Download() error
+}
+
 type goDownloader struct {
 	binDir string
 	arch   string

internal/testarchive/main.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+func main() {
+	// Directories with the _ prefix are ignored by Go. That is important
+	// since the go installation in _bin would include stdlib go modules which would
+	// otherwise cause an error during a build of the CLI.
+	err := createArchive("_build", "_bin", "../..")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error creating archive: %v\n", err)
+		os.Exit(1)
+	}
+}