[runx] Add runx validate, lockfile resolve (#1542)

## Summary

A few major changes to runx:

* Updates newest runx version to allow for github token, validation,
etc.
* lockfile can now resolve runx packages. Exact versions are saved in
lockfile and `install installs deterministic versions according to
lockfile information.
* devpkg can now validate runx packages


## How was it tested?

```bash
devbox add runx:golangci/golangci-lint@latest
devbox add runx:golangci/[email protected]
devbox add runx:golangci/golangci-lint
devbox rm runx:golangci/golangci-lint
```

Inspected devbox.json and devbox.lock in between every command.

Author: mikeland73

.github/workflows/cli-tests.yaml

@@ -37,6 +37,7 @@ defaults:
 
 env:
   HOMEBREW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  DEVBOX_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   HOMEBREW_NO_ANALYTICS: 1
   HOMEBREW_NO_AUTO_UPDATE: 1
   HOMEBREW_NO_EMOJI: 1

devbox.lock

@@ -22,7 +22,8 @@
       }
     },
     "runx:golangci/golangci-lint@latest": {
-      "resolved": "runx:golangci/golangci-lint@latest"
+      "resolved": "golangci/[email protected]",
+      "version": "v1.54.2"
     }
   }
 }

go.mod

@@ -38,7 +38,7 @@ require (
 	github.com/stretchr/testify v1.8.4
 	github.com/wk8/go-ordered-map/v2 v2.1.8
 	github.com/zealic/go2node v0.1.0
-	go.jetpack.io/pkg v0.0.0-20231002215645-9afeb0623fd3
+	go.jetpack.io/pkg v0.0.0-20231006204718-f59feb213022
 	golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17
 	golang.org/x/mod v0.12.0
 	golang.org/x/sync v0.3.0

go.sum

@@ -337,6 +337,8 @@ github.com/zealic/go2node v0.1.0 h1:ofxpve08cmLJBwFdI0lPCk9jfwGWOSD+s6216x0oAaA=
 github.com/zealic/go2node v0.1.0/go.mod h1:GrkFr+HctXwP7vzcU9RsgtAeJjTQ6Ud0IPCQAqpTfBg=
 go.jetpack.io/pkg v0.0.0-20231002215645-9afeb0623fd3 h1:aMydtVCHn7dfotOyV41VAxX5b5OOsCc4TxOXwDt38Yw=
 go.jetpack.io/pkg v0.0.0-20231002215645-9afeb0623fd3/go.mod h1:iaf3e/aENp5luwYFlfCxj+GsiwqHagbvRAY3bIdEgGA=
+go.jetpack.io/pkg v0.0.0-20231006204718-f59feb213022 h1:8TRpo0lYh1Y6zec8Px0yXbnVRXXs+yUPU+TnHNsREdA=
+go.jetpack.io/pkg v0.0.0-20231006204718-f59feb213022/go.mod h1:iaf3e/aENp5luwYFlfCxj+GsiwqHagbvRAY3bIdEgGA=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=

internal/devconfig/env.go

@@ -1,15 +1,20 @@
 package devconfig
 
 import (
+	"context"
+
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/integrations/envsec"
 )
 
-func (c *Config) ComputedEnv(projectDir string) (map[string]string, error) {
+func (c *Config) ComputedEnv(
+	ctx context.Context,
+	projectDir string,
+) (map[string]string, error) {
 	env := map[string]string{}
 	var err error
 	if c.IsEnvsecEnabled() {
-		env, err = envsec.Env(projectDir)
+		env, err = envsec.Env(ctx, projectDir)
 		if err != nil {
 			return nil, err
 		}

internal/devpkg/pkgtype/runx.go

@@ -1,12 +1,39 @@
 package pkgtype
 
-import "strings"
+import (
+	"context"
+	"os"
+	"strings"
+
+	"go.jetpack.io/pkg/sandbox/runx/impl/registry"
+	"go.jetpack.io/pkg/sandbox/runx/impl/runx"
+)
 
 const (
-	RunXScheme = "runx"
-	RunXPrefix = RunXScheme + ":"
+	RunXScheme            = "runx"
+	RunXPrefix            = RunXScheme + ":"
+	githubAPITokenVarName = "DEVBOX_GITHUB_API_TOKEN"
 )
 
+var cachedRegistry *registry.Registry
+
 func IsRunX(s string) bool {
 	return strings.HasPrefix(s, RunXPrefix)
 }
+
+func RunXClient() *runx.RunX {
+	return &runx.RunX{
+		GithubAPIToken: os.Getenv(githubAPITokenVarName),
+	}
+}
+
+func RunXRegistry(ctx context.Context) (*registry.Registry, error) {
+	if cachedRegistry == nil {
+		var err error
+		cachedRegistry, err = registry.NewLocalRegistry(ctx, os.Getenv(githubAPITokenVarName))
+		if err != nil {
+			return nil, err
+		}
+	}
+	return cachedRegistry, nil
+}

internal/devpkg/validation.go

@@ -1,16 +1,17 @@
 package devpkg
 
 import (
+	"context"
 	"strings"
 
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/nix"
 )
 
-func (p *Package) ValidateExists() (bool, error) {
+func (p *Package) ValidateExists(ctx context.Context) (bool, error) {
 	if p.IsRunX() {
-		// TODO implement runx validation
-		return true, nil
+		_, err := p.lockfile.Resolve(p.Raw)
+		return err == nil, err
 	}
 	if p.isVersioned() && p.version() == "" {
 		return false, usererr.New("No version specified for %q.", p.Path)

internal/impl/devbox.go

@@ -22,12 +22,12 @@ import (
 	"github.com/pkg/errors"
 	"github.com/samber/lo"
 	"go.jetpack.io/devbox/internal/devpkg"
+	"go.jetpack.io/devbox/internal/devpkg/pkgtype"
 	"go.jetpack.io/devbox/internal/impl/envpath"
 	"go.jetpack.io/devbox/internal/impl/generate"
 	"go.jetpack.io/devbox/internal/searcher"
 	"go.jetpack.io/devbox/internal/shellgen"
 	"go.jetpack.io/devbox/internal/telemetry"
-	"go.jetpack.io/pkg/sandbox/runx"
 
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/cmdutil"
@@ -851,13 +851,13 @@ func (d *Devbox) computeNixEnv(ctx context.Context, usePrintDevEnvCache bool) (m
 		)
 	}
 
-	env["PATH"], err = d.addUtilitiesToPath(env["PATH"])
+	env["PATH"], err = d.addUtilitiesToPath(ctx, env["PATH"])
 	if err != nil {
 		return nil, err
 	}
 
 	// Include env variables in devbox.json
-	configEnv, err := d.configEnvs(env)
+	configEnv, err := d.configEnvs(ctx, env)
 	if err != nil {
 		return nil, err
 	}
@@ -885,7 +885,7 @@ func (d *Devbox) computeNixEnv(ctx context.Context, usePrintDevEnvCache bool) (m
 	})
 	debug.Log("PATH after filtering with buildInputs (%v) is: %s", buildInputs, nixEnvPath)
 
-	runXPaths, err := d.RunXPaths()
+	runXPaths, err := d.RunXPaths(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -1051,8 +1051,11 @@ func (d *Devbox) checkOldEnvrc() error {
 // their value in the existing env variables. Note, this doesn't
 // allow env variables from outside the shell to be referenced so
 // no leaked variables are caused by this function.
-func (d *Devbox) configEnvs(existingEnv map[string]string) (map[string]string, error) {
-	env, err := d.cfg.ComputedEnv(d.ProjectDir())
+func (d *Devbox) configEnvs(
+	ctx context.Context,
+	existingEnv map[string]string,
+) (map[string]string, error) {
+	env, err := d.cfg.ComputedEnv(ctx, d.ProjectDir())
 	if err != nil {
 		return nil, err
 	}
@@ -1213,11 +1216,15 @@ func (d *Devbox) Lockfile() *lock.File {
 	return d.lockfile
 }
 
-func (d *Devbox) RunXPaths() (string, error) {
+func (d *Devbox) RunXPaths(ctx context.Context) (string, error) {
 	packages := lo.Filter(d.InstallablePackages(), devpkg.IsRunX)
 	paths := []string{}
 	for _, pkg := range packages {
-		p, err := runx.Install(pkg.RunXPath())
+		lockedPkg, err := d.lockfile.Resolve(pkg.Raw)
+		if err != nil {
+			return "", err
+		}
+		p, err := pkgtype.RunXClient().Install(ctx, lockedPkg.Resolved)
 		if err != nil {
 			return "", err
 		}

internal/impl/packages.go

@@ -16,9 +16,9 @@ import (
 	"github.com/pkg/errors"
 	"github.com/samber/lo"
 	"go.jetpack.io/devbox/internal/devpkg"
+	"go.jetpack.io/devbox/internal/devpkg/pkgtype"
 	"go.jetpack.io/devbox/internal/nix/nixprofile"
 	"go.jetpack.io/devbox/internal/shellgen"
-	"go.jetpack.io/pkg/sandbox/runx"
 
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/debug"
@@ -78,7 +78,7 @@ func (d *Devbox) Add(ctx context.Context, platforms, excludePlatforms []string,
 		versionedPkg := devpkg.PackageFromString(pkg.Versioned(), d.lockfile)
 
 		packageNameForConfig := pkg.Raw
-		ok, err := versionedPkg.ValidateExists()
+		ok, err := versionedPkg.ValidateExists(ctx)
 		if (err == nil && ok) || errors.Is(err, devpkg.ErrCannotBuildPackageOnSystem) {
 			// Only use versioned if it exists in search. We can disregard the error
 			// about not building on the current system, since user's can continue
@@ -236,7 +236,7 @@ func (d *Devbox) ensurePackagesAreInstalled(ctx context.Context, mode installMod
 		return err
 	}
 
-	if err := d.InstallRunXPackages(); err != nil {
+	if err := d.InstallRunXPackages(ctx); err != nil {
 		return err
 	}
 
@@ -473,18 +473,17 @@ func resetProfileDirForFlakes(profileDir string) (err error) {
 	return errors.WithStack(os.Remove(profileDir))
 }
 
-func (d *Devbox) InstallRunXPackages() error {
-	for _, pkg := range d.InstallablePackages() {
-		if pkg.IsRunX() {
-			// TODO: Once resolve is implemented, we use whatever version is in the lockfile.
-			if _, err := d.lockfile.Resolve(pkg.Raw); err != nil {
-				return err
-			}
-			_, err := runx.Install(pkg.RunXPath())
-			if err != nil {
-				return fmt.Errorf("error installing runx package %s: %w", pkg, err)
-			}
-
+func (d *Devbox) InstallRunXPackages(ctx context.Context) error {
+	for _, pkg := range lo.Filter(d.InstallablePackages(), devpkg.IsRunX) {
+		lockedPkg, err := d.lockfile.Resolve(pkg.Raw)
+		if err != nil {
+			return err
+		}
+		if _, err := pkgtype.RunXClient().Install(
+			ctx,
+			lockedPkg.Resolved,
+		); err != nil {
+			return fmt.Errorf("error installing runx package %s: %w", pkg, err)
 		}
 	}
 	return nil

internal/impl/util.go

@@ -38,9 +38,12 @@ func (d *Devbox) addDevboxUtilityPackage(ctx context.Context, pkg string) error
 // to (e.g. envsec).
 // Question: Should we add utilityBinPath here? That would allow user to use
 // process-compose, etc
-func (d *Devbox) addUtilitiesToPath(path string) (string, error) {
+func (d *Devbox) addUtilitiesToPath(
+	ctx context.Context,
+	path string,
+) (string, error) {
 	if d.cfg.IsEnvsecEnabled() {
-		envsecPath, err := envsec.EnsureInstalled()
+		envsecPath, err := envsec.EnsureInstalled(ctx)
 		if err != nil {
 			return "", err
 		}