[shell] add packages within shell, leveraging nix-profile (#188)

## Summary

**Motivation:**
We can define a custom nix-profile, and install packages scoped to this
profile.

The goal is to make it easier to add packages when within devbox shell.
Currently,
users need to exit the shell and restart it for the nix package to be
installed - clearly
not ideal. With a nix-profile, if we install the package using that
profile, then the
package's binary should be downloaded and visible within the shell.

Within `devbox shell`:
- previously, we would manually add the path to each package to `PATH`.
- now, we can add the path to the nix-profile to `PATH` and skip
manually adding each package.

**Limitations:**
1. For `devbox add` within a `devbox shell`, the user needs to manually
execute `hash -r` to ensure the latest binaries are visible.
2. ~For `devbox rm` within a `devbox shell`, the user still needs to
restart the shell. I'm not quite sure why the `nix-env --profile
<profile dir> --uninstall <pkg>` isn't working as intended.~ Fixed via
reapplying `nix-env --profile <profile> -if development.nix`
3. `devbox add` and `rm` when in a shell started from a parent directory
will not work.
- A possible fix is to set `DEVBOX_CONFIG_DIR` env-var when inside a
shell, and use that value in `devbox add` and `rm`.
    - Example:
```
> cd testdata/rust/
> devbox shell rust-stable
(devbox)> devbox add go_1_18
Error: No devbox.json found in this directory, or any parent directories. Did you run `devbox init` yet?
```
  



## How was it tested?

- [x] in `testdata/rust/rust-stable`:
  - open `devbox shell`
- verify openssl not installed via `which openssl` and getting
`/usr/bin/openssl`
  - do `devbox add openssl`
- note that `which openssl` still reflects the non nix-store location.
It has NOT yet been updated.
- do `hash -r` and then `which openssl` and see it point to a nix-store
location
  - do `devbox rm openssl`  
- note that `which openssl` still reflects the nix-store location. It
has NOT yet been updated.
- restart `devbox shell` and do `which openssl` to verify it has been
removed

Author: savil

boxcli/add.go

@@ -4,6 +4,8 @@
 package boxcli
 
 import (
+	"os"
+
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"go.jetpack.io/devbox"
@@ -23,10 +25,11 @@ func AddCmd() *cobra.Command {
 
 func addCmdFunc() runFunc {
 	return func(cmd *cobra.Command, args []string) error {
-		box, err := devbox.Open(".")
+		box, err := devbox.Open(".", os.Stdout)
 		if err != nil {
 			return errors.WithStack(err)
 		}
+
 		return box.Add(args...)
 	}
 }

boxcli/build.go

@@ -4,6 +4,8 @@
 package boxcli
 
 import (
+	"os"
+
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"go.jetpack.io/devbox"
@@ -38,7 +40,7 @@ func buildCmdFunc(flags *docker.BuildFlags) runFunc {
 		path := pathArg(args)
 
 		// Check the directory exists.
-		box, err := devbox.Open(path)
+		box, err := devbox.Open(path, os.Stdout)
 		if err != nil {
 			return errors.WithStack(err)
 		}

boxcli/generate.go

@@ -4,6 +4,8 @@
 package boxcli
 
 import (
+	"os"
+
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"go.jetpack.io/devbox"
@@ -23,7 +25,7 @@ func runGenerateCmd(cmd *cobra.Command, args []string) error {
 	path := pathArg(args)
 
 	// Check the directory exists.
-	box, err := devbox.Open(path)
+	box, err := devbox.Open(path, os.Stdout)
 	if err != nil {
 		return errors.WithStack(err)
 	}

boxcli/plan.go

@@ -26,7 +26,7 @@ func runPlanCmd(cmd *cobra.Command, args []string) error {
 	path := pathArg(args)
 
 	// Check the directory exists.
-	box, err := devbox.Open(path)
+	box, err := devbox.Open(path, os.Stdout)
 	if err != nil {
 		return errors.WithStack(err)
 	}

boxcli/rm.go

@@ -4,6 +4,8 @@
 package boxcli
 
 import (
+	"os"
+
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"go.jetpack.io/devbox"
@@ -20,7 +22,7 @@ func RemoveCmd() *cobra.Command {
 }
 
 func runRemoveCmd(cmd *cobra.Command, args []string) error {
-	box, err := devbox.Open(".")
+	box, err := devbox.Open(".", os.Stdout)
 	if err != nil {
 		return errors.WithStack(err)
 	}

boxcli/shell.go

@@ -29,18 +29,15 @@ func runShellCmd(cmd *cobra.Command, args []string) error {
 	path, cmds := parseShellArgs(cmd, args)
 
 	// Check the directory exists.
-	box, err := devbox.Open(path)
+	box, err := devbox.Open(path, os.Stdout)
 	if err != nil {
 		return errors.WithStack(err)
 	}
 
-	inDevboxShell := os.Getenv("DEVBOX_SHELL_ENABLED")
-	if inDevboxShell != "" && inDevboxShell != "0" && inDevboxShell != "false" {
+	if devbox.IsDevboxShellEnabled() {
 		return errors.New("You are already in an active devbox shell.\nRun 'exit' before calling devbox shell again. Shell inception is not supported.")
 	}
 
-	fmt.Println("Installing nix packages. This may take a while...")
-
 	if len(cmds) > 0 {
 		err = box.Exec(cmds...)
 	} else {

devbox.go

@@ -6,8 +6,12 @@ package devbox
 
 import (
 	"fmt"
+	"io"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"strconv"
+	"strings"
 
 	"github.com/pkg/errors"
 	"go.jetpack.io/devbox/boxcli/usererr"
@@ -21,6 +25,9 @@ import (
 	"golang.org/x/exp/slices"
 )
 
+// profileDir contains the contents of the profile generated via `nix-env --profile profileDir <command>`
+const profileDir = ".devbox/profile"
+
 // configFilename is name of the JSON file that defines a devbox environment.
 const configFilename = "devbox.json"
 
@@ -37,10 +44,11 @@ type Devbox struct {
 	cfg *Config
 	// srcDir is the directory where the config file (devbox.json) resides
 	srcDir string
+	writer io.Writer
 }
 
 // Open opens a devbox by reading the config file in dir.
-func Open(dir string) (*Devbox, error) {
+func Open(dir string, writer io.Writer) (*Devbox, error) {
 
 	cfgDir, err := findConfigDir(dir)
 	if err != nil {
@@ -56,6 +64,7 @@ func Open(dir string) (*Devbox, error) {
 	box := &Devbox{
 		cfg:    cfg,
 		srcDir: cfgDir,
+		writer: writer,
 	}
 	return box, nil
 }
@@ -72,22 +81,36 @@ func (d *Devbox) Add(pkgs ...string) error {
 		}
 	}
 
-	// Add to Packages only if it's not already there
+	// Add to Packages to config only if it's not already there
 	for _, pkg := range pkgs {
 		if slices.Contains(d.cfg.Packages, pkg) {
 			continue
 		}
 		d.cfg.Packages = append(d.cfg.Packages, pkg)
 	}
-	return d.saveCfg()
+	if err := d.saveCfg(); err != nil {
+		return err
+	}
+
+	if err := d.ensurePackagesAreInstalled(install); err != nil {
+		return err
+	}
+	return d.printPackageUpdateMessage(install, pkgs)
 }
 
 // Remove removes Nix packages from the config so that it no longer exists in
 // the devbox environment.
 func (d *Devbox) Remove(pkgs ...string) error {
 	// Remove packages from config.
 	d.cfg.Packages = pkgslice.Exclude(d.cfg.Packages, pkgs)
-	return d.saveCfg()
+	if err := d.saveCfg(); err != nil {
+		return err
+	}
+
+	if err := d.ensurePackagesAreInstalled(uninstall); err != nil {
+		return err
+	}
+	return d.printPackageUpdateMessage(uninstall, pkgs)
 }
 
 // Build creates a Docker image containing a shell with the devbox environment.
@@ -142,35 +165,33 @@ func (d *Devbox) Generate() error {
 // Shell generates the devbox environment and launches nix-shell as a child
 // process.
 func (d *Devbox) Shell() error {
-	if err := d.generateShellFiles(); err != nil {
-		return errors.WithStack(err)
+
+	if err := d.ensurePackagesAreInstalled(install); err != nil {
+		return err
 	}
+
 	plan, err := d.ShellPlan()
 	if err != nil {
 		return errors.WithStack(err)
 	}
-	nixDir := filepath.Join(d.srcDir, ".devbox/gen/shell.nix")
-	sh, err := nix.DetectShell(nix.WithPlanInitHook(plan.ShellInitHook))
+	nixShellFilePath := filepath.Join(d.srcDir, ".devbox/gen/shell.nix")
+	sh, err := nix.DetectShell(nix.WithPlanInitHook(plan.ShellInitHook), nix.WithProfile(d.profileDir()))
 	if err != nil {
 		// Fall back to using a plain Nix shell.
 		sh = &nix.Shell{}
 	}
 	sh.UserInitHook = d.cfg.Shell.InitHook.String()
-	return sh.Run(nixDir)
+	return sh.Run(nixShellFilePath)
 }
 
 func (d *Devbox) Exec(cmds ...string) error {
-	plan, err := d.ShellPlan()
-	if err != nil {
-		return errors.WithStack(err)
-	}
-	if plan.Invalid() {
-		return plan.Error()
-	}
-	err = generate(d.srcDir, plan, shellFiles)
-	if err != nil {
-		return errors.WithStack(err)
+	if err := d.ensurePackagesAreInstalled(install); err != nil {
+		return err
 	}
+
+	pathWithProfileBin := fmt.Sprintf("PATH=%s:$PATH", d.profileBinDir())
+	cmds = append([]string{pathWithProfileBin}, cmds...)
+
 	nixDir := filepath.Join(d.srcDir, ".devbox/gen/shell.nix")
 	return nix.Exec(nixDir, cmds)
 }
@@ -226,6 +247,14 @@ func (d *Devbox) generateBuildFiles() error {
 	return generate(d.srcDir, buildPlan, buildFiles)
 }
 
+func (d *Devbox) profileDir() string {
+	return filepath.Join(d.srcDir, profileDir)
+}
+
+func (d *Devbox) profileBinDir() string {
+	return filepath.Join(d.profileDir(), "bin")
+}
+
 func missingDevboxJSONError(dir string) error {
 
 	// We try to prettify the `dir` before printing
@@ -266,3 +295,79 @@ func findConfigDir(dir string) (string, error) {
 	}
 	return "", missingDevboxJSONError(dir)
 }
+
+// installMode is an enum for helping with ensurePackagesAreInstalled implementation
+type installMode string
+
+const (
+	install   installMode = "install"
+	uninstall installMode = "uninstall"
+)
+
+func (d *Devbox) ensurePackagesAreInstalled(mode installMode) error {
+	if err := d.Generate(); err != nil {
+		return err
+	}
+
+	installingVerb := "Installing"
+	if mode == uninstall {
+		installingVerb = "Uninstalling"
+	}
+	fmt.Fprintf(d.writer, "%s nix packages. This may take a while...", installingVerb)
+
+	// We need to re-install the packages
+	if err := d.ApplyDevNixDerivation(); err != nil {
+		fmt.Println()
+		return err
+	}
+	fmt.Println("done.")
+
+	return nil
+}
+
+func (d *Devbox) printPackageUpdateMessage(mode installMode, pkgs []string) error {
+	// (Only when in devbox shell) Prompt the user to run `hash -r` to ensure their
+	// shell can access the most recently installed binaries, or ensure their
+	// recently uninstalled binaries are not accidentally still available.
+	if len(pkgs) > 0 && IsDevboxShellEnabled() {
+		installedVerb := "installed"
+		if mode == uninstall {
+			installedVerb = "removed"
+		}
+
+		successMsg := fmt.Sprintf("%s is now %s.", pkgs[0], installedVerb)
+		if len(pkgs) > 1 {
+			successMsg = fmt.Sprintf("%s are now %s.", strings.Join(pkgs, ", "), installedVerb)
+		}
+		fmt.Fprint(d.writer, successMsg)
+		fmt.Fprintln(d.writer, " Run `hash -r` to ensure your shell is updated.")
+	}
+	return nil
+}
+
+// ApplyDevNixDerivation ensures the local profile has exactly the packages in the development.nix file
+//
+// Will move to a store interface/package
+func (d *Devbox) ApplyDevNixDerivation() error {
+
+	cmdStr := fmt.Sprintf(
+		"--profile %s --install -f %s/.devbox/gen/development.nix",
+		filepath.Join(d.srcDir, profileDir),
+		d.srcDir,
+	)
+	cmdParts := strings.Split(cmdStr, " ")
+	execCmd := exec.Command("nix-env", cmdParts...)
+
+	debug.Log("running command: %s\n", execCmd.Args)
+	err := execCmd.Run()
+	return errors.WithStack(err)
+}
+
+// Move to a utility package?
+func IsDevboxShellEnabled() bool {
+	inDevboxShell, err := strconv.ParseBool(os.Getenv("DEVBOX_SHELL_ENABLED"))
+	if err != nil {
+		return false
+	}
+	return inDevboxShell
+}

devbox_test.go

@@ -39,7 +39,7 @@ func testExample(t *testing.T, testPath string) {
 		goldenFile := filepath.Join(baseDir, "plan.json")
 		hasGoldenFile := fileExists(goldenFile)
 
-		box, err := Open(baseDir)
+		box, err := Open(baseDir, os.Stdout)
 		assert.NoErrorf(err, "%s should be a valid devbox project", baseDir)
 
 		// Just for tests, we make srcDir be a relative path so that the paths in plan.json

generate.go

@@ -20,7 +20,7 @@ import (
 //go:embed tmpl/* tmpl/.*
 var tmplFS embed.FS
 
-var shellFiles = []string{".gitignore", "shell.nix"}
+var shellFiles = []string{".gitignore", "development.nix", "shell.nix"}
 var buildFiles = []string{".gitignore", "development.nix", "runtime.nix", "Dockerfile", "Dockerfile.dockerignore"}
 
 func generate(rootPath string, plan *plansdk.Plan, files []string) error {