[nix] Create bin wrappers. Eliminate need to refresh (#803)

## Summary

This creates bin wrappers for all binaries and services. It sources the
devbox environment in each wrapper and in service wrappers it sets
environment variables if not already set.

Benefits:

* No longer need to call "refresh" after installing a package.
* Fix bug where env variables get overwritten on refresh
* Can pass limited flags to wrappers (basically only at the end)

Downsides (fixable):
* minor performance hit. Roughly 5ms are added per wrapper. We might be
able to fix this in the future by turning wrappers into sym links if
nothing needs to be sourced.

Future benefits:
* Plugin env variables can be removed from devbox env

## How was it tested?

```
devbox shell
devbox add apacheHttpd
HTTPD_PORT=8085 devbox services start apache
curl localhost:8085
```

Author: mikeland73

devbox.go

@@ -18,7 +18,7 @@ type Devbox interface {
 	// Add adds a Nix package to the config so that it's available in the devbox
 	// environment. It validates that the Nix package exists, but doesn't install
 	// it. Adding a duplicate package is a no-op.
-	Add(pkgs ...string) error
+	Add(ctx context.Context, pkgs ...string) error
 	AddGlobal(pkgs ...string) error
 	Config() *impl.Config
 	ProjectDir() string
@@ -35,12 +35,12 @@ type Devbox interface {
 	PullGlobal(path string) error
 	// Remove removes Nix packages from the config so that it no longer exists in
 	// the devbox environment.
-	Remove(pkgs ...string) error
+	Remove(ctx context.Context, pkgs ...string) error
 	RemoveGlobal(pkgs ...string) error
 	RunScript(scriptName string, scriptArgs []string) error
 	Services() (plugin.Services, error)
 	// Shell generates the devbox environment and launches nix-shell as a child process.
-	Shell() error
+	Shell(ctx context.Context) error
 	// ShellPlan creates a plan of the actions that devbox will take to generate its
 	// shell environment.
 	ShellPlan() (*plansdk.ShellPlan, error)

internal/boxcli/add.go

@@ -54,5 +54,5 @@ func addCmdFunc(cmd *cobra.Command, args []string, flags addCmdFlags) error {
 		return errors.WithStack(err)
 	}
 
-	return box.Add(args...)
+	return box.Add(cmd.Context(), args...)
 }

internal/boxcli/install.go

@@ -0,0 +1,33 @@
+package boxcli
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+func InstallCmd() *cobra.Command {
+	flags := runCmdFlags{}
+	command := &cobra.Command{
+		Use:   "install",
+		Short: "Installs all packages mentioned in devbox.json",
+		Long: "Starts a new devbox shell and installs all packages mentioned in devbox.json in current directory or" +
+			"a directory specified via --config. \n\n Then exits the shell when packages are done installing.\n\n ",
+		Args:    cobra.MaximumNArgs(0),
+		PreRunE: ensureNixInstalled,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			// the colon ':' character in standard shell means noop.
+			// So essentially, this command is running devbox run noop
+			err := runScriptCmd(cmd, []string{":"}, flags)
+			if err != nil {
+				return err
+			}
+			fmt.Fprintln(cmd.ErrOrStderr(), "Finished installing packages.")
+			return nil
+		},
+	}
+
+	flags.config.register(command)
+
+	return command
+}

internal/boxcli/rm.go

@@ -35,5 +35,5 @@ func runRemoveCmd(cmd *cobra.Command, args []string, flags removeCmdFlags) error
 		return errors.WithStack(err)
 	}
 
-	return box.Remove(args...)
+	return box.Remove(cmd.Context(), args...)
 }

internal/boxcli/run.go

@@ -4,8 +4,6 @@
 package boxcli
 
 import (
-	"fmt"
-
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"go.jetpack.io/devbox"
@@ -43,32 +41,6 @@ func RunCmd() *cobra.Command {
 	return command
 }
 
-func InstallCmd() *cobra.Command {
-	flags := runCmdFlags{}
-	command := &cobra.Command{
-		Use:   "install",
-		Short: "Installs all packages mentioned in devbox.json",
-		Long: "Starts a new devbox shell and installs all packages mentioned in devbox.json in current directory or" +
-			"a directory specified via --config. \n\n Then exits the shell when packages are done installing.\n\n ",
-		Args:    cobra.MaximumNArgs(0),
-		PreRunE: ensureNixInstalled,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			// the colon ':' character in standard shell means noop.
-			// So essentially, this command is running devbox run noop
-			err := runScriptCmd(cmd, []string{":"}, flags)
-			if err != nil {
-				return err
-			}
-			fmt.Fprintln(cmd.ErrOrStderr(), "Finished installing packages.")
-			return nil
-		},
-	}
-
-	flags.config.register(command)
-
-	return command
-}
-
 func listScripts(cmd *cobra.Command, flags runCmdFlags) []string {
 	path, err := configPathFromUser([]string{}, &flags.config)
 	if err != nil {

internal/boxcli/shell.go

@@ -66,7 +66,7 @@ func runShellCmd(cmd *cobra.Command, args []string, flags shellCmdFlags) error {
 		return shellInceptionErrorMsg("devbox shell")
 	}
 
-	return box.Shell()
+	return box.Shell(cmd.Context())
 }
 
 func parseShellArgs(cmd *cobra.Command, args []string, flags shellCmdFlags) (string, []string, error) {

internal/boxcli/shellenv.go

@@ -27,7 +27,6 @@ func shellEnvCmd() *cobra.Command {
 				return err
 			}
 			fmt.Fprintln(cmd.OutOrStdout(), s)
-			fmt.Fprintln(cmd.OutOrStdout(), ";hash -r")
 			return nil
 		},
 	}

internal/impl/config.go

@@ -19,7 +19,6 @@ import (
 	"go.jetpack.io/devbox/internal/debug"
 	"go.jetpack.io/devbox/internal/impl/shellcmd"
 	"go.jetpack.io/devbox/internal/planner/plansdk"
-	"go.jetpack.io/devbox/internal/ux"
 )
 
 // Config defines a devbox environment as JSON.
@@ -56,7 +55,6 @@ type Stage struct {
 func (c *Config) Packages(w io.Writer) []string {
 	dataPath, err := GlobalDataPath()
 	if err != nil {
-		ux.Ferror(w, "unable to get devbox global data path: %s\n", err)
 		return c.RawPackages
 	}
 	global, err := readConfig(filepath.Join(dataPath, "devbox.json"))

internal/impl/devbox.go

@@ -35,6 +35,7 @@ import (
 	"go.jetpack.io/devbox/internal/services"
 	"go.jetpack.io/devbox/internal/telemetry"
 	"go.jetpack.io/devbox/internal/ux"
+	"go.jetpack.io/devbox/internal/wrapnix"
 )
 
 const (
@@ -148,8 +149,8 @@ func (d *Devbox) Generate() error {
 	return errors.WithStack(d.generateShellFiles())
 }
 
-func (d *Devbox) Shell() error {
-	ctx, task := trace.NewTask(context.Background(), "devboxShell")
+func (d *Devbox) Shell(ctx context.Context) error {
+	ctx, task := trace.NewTask(ctx, "devboxShell")
 	defer task.End()
 
 	if err := d.ensurePackagesAreInstalled(ctx, ensure); err != nil {
@@ -167,11 +168,15 @@ func (d *Devbox) Shell() error {
 		return err
 	}
 
-	env, err := d.computeNixEnv(ctx)
+	env, err := d.cachedComputeNixEnv(ctx)
 	if err != nil {
 		return err
 	}
 
+	if err := wrapnix.CreateWrappers(ctx, d); err != nil {
+		return err
+	}
+
 	shellStartTime := os.Getenv("DEVBOX_SHELL_START_TIME")
 	if shellStartTime == "" {
 		shellStartTime = telemetry.UnixTimestampFromTime(telemetry.CommandStartTime())
@@ -207,11 +212,15 @@ func (d *Devbox) RunScript(cmdName string, cmdArgs []string) error {
 		return err
 	}
 
-	env, err := d.computeNixEnv(ctx)
+	env, err := d.cachedComputeNixEnv(ctx)
 	if err != nil {
 		return err
 	}
 
+	if err = wrapnix.CreateWrappers(ctx, d); err != nil {
+		return err
+	}
+
 	var cmdWithArgs []string
 	if _, ok := d.cfg.Shell.Scripts[cmdName]; ok {
 		// it's a script, so replace the command with the script file's path.
@@ -247,7 +256,7 @@ func (d *Devbox) PrintEnv() (string, error) {
 	ctx, task := trace.NewTask(context.Background(), "devboxPrintEnv")
 	defer task.End()
 
-	envs, err := d.computeNixEnv(ctx)
+	envs, err := d.cachedComputeNixEnv(ctx)
 	if err != nil {
 		return "", err
 	}
@@ -544,21 +553,26 @@ func (d *Devbox) computeNixEnv(ctx context.Context) (map[string]string, error) {
 	debug.Log("nix environment PATH is: %s", env)
 
 	// Add any vars defined in plugins.
+	// TODO: Now that we have bin wrappers, this may can eventually be removed.
+	// We still need to be able to add env variables to non-service binaries
+	// (e.g. ruby). This would involve understanding what binaries are associated
+	// to a given plugin.
 	pluginEnv, err := plugin.Env(d.mergedPackages(), d.projectDir, env)
 	if err != nil {
 		return nil, err
 	}
 
-	for k, v := range pluginEnv {
-		env[k] = v
-	}
+	addEnvIfNotPreviouslySetByDevbox(env, pluginEnv)
+
+	// Prepend virtenv bin path first so user can override it if needed. Virtenv
+	// is where the bin wrappers live
+	env["PATH"] = JoinPathLists(d.virtenvBinPath(), env["PATH"])
 
 	// Include env variables in devbox.json
-	if featureflag.EnvConfig.Enabled() {
-		for k, v := range d.configEnvs(env) {
-			env[k] = v
-		}
-	}
+	configEnv := d.configEnvs(env)
+	addEnvIfNotPreviouslySetByDevbox(env, configEnv)
+
+	markEnvsAsSetByDevbox(pluginEnv, configEnv)
 
 	nixEnvPath := env["PATH"]
 	debug.Log("PATH after plugins and config is: %s", nixEnvPath)
@@ -571,6 +585,17 @@ func (d *Devbox) computeNixEnv(ctx context.Context) (map[string]string, error) {
 	return env, nil
 }
 
+var nixEnvCache map[string]string
+
+// cachedComputeNixEnv is a wrapper around computeNixEnv that caches the result.
+func (d *Devbox) cachedComputeNixEnv(ctx context.Context) (map[string]string, error) {
+	var err error
+	if nixEnvCache == nil {
+		nixEnvCache, err = d.computeNixEnv(ctx)
+	}
+	return nixEnvCache, err
+}
+
 // writeScriptsToFiles writes scripts defined in devbox.json into files inside .devbox/gen/scripts.
 // Scripts (and hooks) are persisted so that we can easily call them from devbox run (inside or outside shell).
 func (d *Devbox) writeScriptsToFiles() error {
@@ -768,3 +793,37 @@ func (d *Devbox) setCommonHelperEnvVars(env map[string]string) {
 	env["LD_LIBRARY_PATH"] = filepath.Join(d.projectDir, nix.ProfilePath, "lib") + ":" + env["LD_LIBRARY_PATH"]
 	env["LIBRARY_PATH"] = filepath.Join(d.projectDir, nix.ProfilePath, "lib") + ":" + env["LIBRARY_PATH"]
 }
+
+func (d *Devbox) virtenvBinPath() string {
+	return filepath.Join(d.projectDir, plugin.VirtenvBinPath)
+}
+
+// nix bins returns the paths to all the nix binaries that are installed by
+// the flake. If there are conflicts, it returns the first one it finds of a
+// give name. This matches how nix flakes behaves if there are conflicts in
+// buildInputs
+func (d *Devbox) NixBins(ctx context.Context) ([]string, error) {
+	env, err := d.cachedComputeNixEnv(ctx)
+
+	if err != nil {
+		return nil, err
+	}
+	dirs := strings.Split(env["buildInputs"], " ")
+	bins := map[string]string{}
+	for _, dir := range dirs {
+		binPath := filepath.Join(dir, "bin")
+		if _, err = os.Stat(binPath); os.IsNotExist(err) {
+			continue
+		}
+		files, err := os.ReadDir(binPath)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+		for _, file := range files {
+			if _, alreadySet := bins[file.Name()]; !alreadySet {
+				bins[file.Name()] = filepath.Join(binPath, file.Name())
+			}
+		}
+	}
+	return lo.Values(bins), nil
+}

internal/impl/envvars.go

@@ -6,6 +6,8 @@ import (
 	"strings"
 )
 
+const devboxSetPrefix = "__DEVBOX_SET_"
+
 func mapToPairs(m map[string]string) []string {
 	pairs := []string{}
 	for k, v := range m {
@@ -53,3 +55,25 @@ func exportify(vars map[string]string) string {
 	}
 	return strings.TrimSpace(strb.String())
 }
+
+// addEnvIfNotPreviouslySetByDevbox adds the key-value pairs from new to existing,
+// but only if the key was not previously set by devbox
+// Caveat, this won't mark the values as set by devbox automatically. Instead,
+// you need to call markEnvAsSetByDevbox when you are done setting variables.
+// This is so you can add variables from multiple sources (e.g. plugin, devbox.json)
+// that may build on each other (e.g. PATH=$PATH:...)
+func addEnvIfNotPreviouslySetByDevbox(existing, new map[string]string) {
+	for k, v := range new {
+		if _, alreadySet := existing[devboxSetPrefix+k]; !alreadySet {
+			existing[k] = v
+		}
+	}
+}
+
+func markEnvsAsSetByDevbox(envs ...map[string]string) {
+	for _, env := range envs {
+		for key := range env {
+			env[devboxSetPrefix+key] = "1"
+		}
+	}
+}