Added python.DetectInterpreters and other utils (#805)

This PR adds a few utilities related to Python interpreter detection:

- `python.DetectInterpreters` to detect all Python versions available in
`$PATH` by executing every matched binary name with `--version` flag.
- `python.DetectVirtualEnvPath` to detect if there's any child virtual
environment in `src` directory
- `python.DetectExecutable` to detect if there's python3 installed
either by `which python3` command or by calling
`python.DetectInterpreters().AtLeast("v3.8")`

To be merged after #804, as one of
the steps to get #637 in, as
previously discussed.

Author: nfx

bundle/artifacts/whl/build.go

@@ -9,7 +9,7 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/libs/cmdio"
-	"github.com/databricks/cli/python"
+	"github.com/databricks/cli/libs/python"
 )
 
 type build struct {

bundle/artifacts/whl/infer.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 
 	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/python"
+	"github.com/databricks/cli/libs/python"
 )
 
 type infer struct {

libs/python/detect.go

@@ -0,0 +1,34 @@
+package python
+
+import (
+	"context"
+	"errors"
+	"os/exec"
+)
+
+func DetectExecutable(ctx context.Context) (string, error) {
+	// TODO: add a shortcut if .python-version file is detected somewhere in
+	// the parent directory tree.
+	//
+	// See https://github.com/pyenv/pyenv#understanding-python-version-selection
+	out, err := exec.LookPath("python3")
+	// most of the OS'es have python3 in $PATH, but for those which don't,
+	// we perform the latest version lookup
+	if err != nil && !errors.Is(err, exec.ErrNotFound) {
+		return "", err
+	}
+	if out != "" {
+		return out, nil
+	}
+	// otherwise, detect all interpreters and pick the least that satisfies
+	// minimal version requirements
+	all, err := DetectInterpreters(ctx)
+	if err != nil {
+		return "", err
+	}
+	interpreter, err := all.AtLeast("3.8")
+	if err != nil {
+		return "", err
+	}
+	return interpreter.Path, nil
+}

libs/python/detect_unix_test.go

@@ -0,0 +1,39 @@
+//go:build unix
+
+package python
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDetectsViaPathLookup(t *testing.T) {
+	ctx := context.Background()
+	py, err := DetectExecutable(ctx)
+	assert.NoError(t, err)
+	assert.NotEmpty(t, py)
+}
+
+func TestDetectsViaListing(t *testing.T) {
+	t.Setenv("PATH", "testdata/other-binaries-filtered")
+	ctx := context.Background()
+	py, err := DetectExecutable(ctx)
+	assert.NoError(t, err)
+	assert.Equal(t, "testdata/other-binaries-filtered/python3.10", py)
+}
+
+func TestDetectFailsNoInterpreters(t *testing.T) {
+	t.Setenv("PATH", "testdata")
+	ctx := context.Background()
+	_, err := DetectExecutable(ctx)
+	assert.Equal(t, ErrNoPythonInterpreters, err)
+}
+
+func TestDetectFailsNoMinimalVersion(t *testing.T) {
+	t.Setenv("PATH", "testdata/no-python3")
+	ctx := context.Background()
+	_, err := DetectExecutable(ctx)
+	assert.EqualError(t, err, "cannot find Python greater or equal to v3.8.0")
+}

libs/python/detect_win_test.go

@@ -0,0 +1,24 @@
+//go:build windows
+
+package python
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDetectsViaPathLookup(t *testing.T) {
+	ctx := context.Background()
+	py, err := DetectExecutable(ctx)
+	assert.NoError(t, err)
+	assert.NotEmpty(t, py)
+}
+
+func TestDetectFailsNoInterpreters(t *testing.T) {
+	t.Setenv("PATH", "testdata")
+	ctx := context.Background()
+	_, err := DetectExecutable(ctx)
+	assert.ErrorIs(t, err, ErrNoPythonInterpreters)
+}

libs/python/interpreters.go

@@ -0,0 +1,216 @@
+package python
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sort"
+	"strings"
+
+	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/cli/libs/process"
+	"golang.org/x/mod/semver"
+)
+
+var ErrNoPythonInterpreters = errors.New("no python3 interpreters found")
+
+const officialMswinPython = "(Python Official) https://python.org/downloads/windows"
+const microsoftStorePython = "(Microsoft Store) https://apps.microsoft.com/store/search?publisher=Python%20Software%20Foundation"
+
+const worldWriteable = 0o002
+
+type Interpreter struct {
+	Version string
+	Path    string
+}
+
+func (i Interpreter) String() string {
+	return fmt.Sprintf("%s (%s)", i.Version, i.Path)
+}
+
+type allInterpreters []Interpreter
+
+func (a allInterpreters) Latest() Interpreter {
+	return a[len(a)-1]
+}
+
+func (a allInterpreters) AtLeast(minimalVersion string) (*Interpreter, error) {
+	canonicalMinimalVersion := semver.Canonical("v" + strings.TrimPrefix(minimalVersion, "v"))
+	if canonicalMinimalVersion == "" {
+		return nil, fmt.Errorf("invalid SemVer: %s", minimalVersion)
+	}
+	for _, interpreter := range a {
+		cmp := semver.Compare(interpreter.Version, canonicalMinimalVersion)
+		if cmp < 0 {
+			continue
+		}
+		return &interpreter, nil
+	}
+	return nil, fmt.Errorf("cannot find Python greater or equal to %s", canonicalMinimalVersion)
+}
+
+func DetectInterpreters(ctx context.Context) (allInterpreters, error) {
+	found := allInterpreters{}
+	seen := map[string]bool{}
+	executables, err := pythonicExecutablesFromPathEnvironment(ctx)
+	if err != nil {
+		return nil, err
+	}
+	log.Debugf(ctx, "found %d potential alternative Python versions in $PATH", len(executables))
+	for _, resolved := range executables {
+		if seen[resolved] {
+			continue
+		}
+		seen[resolved] = true
+		// probe the binary version by executing it, like `python --version`
+		// and parsing the output.
+		//
+		// Keep in mind, that mswin installations get python.exe and pythonw.exe,
+		// which are slightly different: see https://stackoverflow.com/a/30313091
+		out, err := process.Background(ctx, []string{resolved, "--version"})
+		var processErr *process.ProcessError
+		if errors.As(err, &processErr) {
+			log.Debugf(ctx, "failed to check version for %s: %s", resolved, processErr.Err)
+			continue
+		}
+		if err != nil {
+			log.Debugf(ctx, "failed to check version for %s: %s", resolved, err)
+			continue
+		}
+		version := validPythonVersion(ctx, resolved, out)
+		if version == "" {
+			continue
+		}
+		found = append(found, Interpreter{
+			Version: version,
+			Path:    resolved,
+		})
+	}
+	if runtime.GOOS == "windows" && len(found) == 0 {
+		return nil, fmt.Errorf("%w. Install them from %s or %s and restart the shell",
+			ErrNoPythonInterpreters, officialMswinPython, microsoftStorePython)
+	}
+	if len(found) == 0 {
+		return nil, ErrNoPythonInterpreters
+	}
+	sort.Slice(found, func(i, j int) bool {
+		a := found[i].Version
+		b := found[j].Version
+		cmp := semver.Compare(a, b)
+		if cmp != 0 {
+			return cmp < 0
+		}
+		return a < b
+	})
+	return found, nil
+}
+
+func pythonicExecutablesFromPathEnvironment(ctx context.Context) (out []string, err error) {
+	paths := strings.Split(os.Getenv("PATH"), string(os.PathListSeparator))
+	for _, prefix := range paths {
+		info, err := os.Stat(prefix)
+		if errors.Is(err, fs.ErrNotExist) {
+			// some directories in $PATH may not exist
+			continue
+		}
+		if errors.Is(err, fs.ErrPermission) {
+			// some directories we cannot list
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("stat %s: %w", prefix, err)
+		}
+		if !info.IsDir() {
+			continue
+		}
+		perm := info.Mode().Perm()
+		if runtime.GOOS != "windows" && perm&worldWriteable != 0 {
+			// we try not to run any python binary that sits in a writable folder by all users.
+			// this is mainly to avoid breaking the security model on a multi-user system.
+			// If the PATH is pointing somewhere untrusted it is the user fault, but we can
+			// help here.
+			//
+			// See https://github.com/databricks/cli/pull/805#issuecomment-1735403952
+			log.Debugf(ctx, "%s is world-writeable (%s), skipping for security reasons", prefix, perm)
+			continue
+		}
+		entries, err := os.ReadDir(prefix)
+		if errors.Is(err, fs.ErrPermission) {
+			// some directories we cannot list
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("listing %s: %w", prefix, err)
+		}
+		for _, v := range entries {
+			if v.IsDir() {
+				continue
+			}
+			if strings.Contains(v.Name(), "-") {
+				// skip python3-config, python3.10-config, etc
+				continue
+			}
+			// If Python3 is installed on Windows through GUI installer app that was
+			// downloaded from https://python.org/downloads/windows, it may appear
+			// in $PATH as `python`, even though it means Python 2.7 in all other
+			// operating systems (macOS, Linux).
+			//
+			// See https://github.com/databrickslabs/ucx/issues/281
+			if !strings.HasPrefix(v.Name(), "python") {
+				continue
+			}
+			bin := filepath.Join(prefix, v.Name())
+			resolved, err := filepath.EvalSymlinks(bin)
+			if err != nil {
+				log.Debugf(ctx, "cannot resolve symlink for %s: %s", bin, resolved)
+				continue
+			}
+			out = append(out, resolved)
+		}
+	}
+	return out, nil
+}
+
+func validPythonVersion(ctx context.Context, resolved, out string) string {
+	out = strings.TrimSpace(out)
+	log.Debugf(ctx, "%s --version: %s", resolved, out)
+
+	words := strings.Split(out, " ")
+	// The Python distribution from the Windows Store is available in $PATH as `python.exe`
+	// and `python3.exe`, even though it symlinks to a real file packaged with some versions of Windows:
+	// /c/Program Files/WindowsApps/Microsoft.DesktopAppInstaller_.../AppInstallerPythonRedirector.exe.
+	// Executing the `python` command from this distribution opens the Windows Store, allowing users to
+	// download and install Python. Once installed, it replaces the `python.exe` and `python3.exe`` stub
+	// with the genuine Python executable. Additionally, once user installs from the main installer at
+	// https://python.org/downloads/windows, it does not replace this stub.
+	//
+	// However, a drawback is that if this initial stub is run with any command line arguments, it quietly
+	// fails to execute. According to https://github.com/databrickslabs/ucx/issues/281, it can be
+	// detected by seeing just the "Python" output without any version info from the `python --version`
+	// command execution.
+	//
+	// See https://github.com/pypa/packaging-problems/issues/379
+	// See https://bugs.python.org/issue41327
+	if len(words) < 2 {
+		log.Debugf(ctx, "%s --version: stub from Windows Store", resolved)
+		return ""
+	}
+
+	if words[0] != "Python" {
+		log.Debugf(ctx, "%s --version: not a Python", resolved)
+		return ""
+	}
+
+	lastWord := words[len(words)-1]
+	version := semver.Canonical("v" + lastWord)
+	if version == "" {
+		log.Debugf(ctx, "%s --version: invalid SemVer: %s", resolved, lastWord)
+		return ""
+	}
+
+	return version
+}