[global] Unify nix profile install (#695)

## Summary

This fixes a few global issues:

* stdout not being attached when doing global install.
* Prefetching (and printing) nixpkgs when needed

It does so by refactoring some code around:

* Unify different ways of installing nix packages (local, global, and
utility all use nix.ProfileInstall)
* We now always check if nixpkgs is installed, improving messaging when
we need to install it.
* Moved some nix related stuff to nix packages (calling `nix profile
install`, packageInstallIgnore, checking if nixpkgs is installed)

What this does not fix: If we install a very large package (e.g.
`bazel`) we block on `cmd.Run()` and don't print any output. This
affects all ways of installing.

## How was it tested?

```bash
devbox global add go_1_20 hello nginx
devbox add go_1_19
devbox shell
```

Also tried changing hashes on global devbox.json, installing conflicting
binaries, adding and removing large packages (e.g. pulumi, bazel, etc).
Bazel was a bad experience.

Author: mikeland73

.github/workflows/cli-tests.yaml

@@ -99,7 +99,7 @@ jobs:
           echo "NIX_SSL_CERT_FILE=$NIX_SSL_CERT_FILE" >> $GITHUB_ENV
           echo "PATH=$PATH" >> $GITHUB_ENV
       - name: Run tests
-        run: go test -race -cover ./...
+        run: go test -race -cover -v ./...
 
   auto-nix-install: # ensure Devbox installs nix and works properly after installation.
     runs-on: ubuntu-latest
@@ -114,4 +114,4 @@ jobs:
       - name: Install nix and devbox packages
         run: |
           NIX_INSTALLER_NO_CHANNEL_ADD=1
-          devbox shell -- echo "Installing packages..."
+          devbox run echo "Installing packages..."

internal/impl/devbox.go

@@ -629,7 +629,7 @@ func (d *Devbox) StartProcessManager(ctx context.Context) error {
 	processComposePath, err := utilityLookPath("process-compose")
 	if err != nil {
 		fmt.Fprintln(d.writer, "Installing process-compose. This may take a minute but will only happen once.")
-		if err = addDevboxUtilityPackage("process-compose"); err != nil {
+		if err = d.addDevboxUtilityPackage("process-compose"); err != nil {
 			return err
 		}
 	}
@@ -868,7 +868,7 @@ func (d *Devbox) installNixProfile() (err error) {
 	)
 
 	cmd.Env = nix.DefaultEnv()
-	cmd.Stdout = &nixPackageInstallWriter{d.writer}
+	cmd.Stdout = &nix.PackageInstallWriter{Writer: d.writer}
 
 	cmd.Stderr = cmd.Stdout
 

internal/impl/global.go

@@ -42,8 +42,18 @@ func (d *Devbox) AddGlobal(pkgs ...string) error {
 	if err != nil {
 		return err
 	}
-	for _, pkg := range pkgs {
-		if err := nix.ProfileInstall(profilePath, plansdk.DefaultNixpkgsCommit, pkg); err != nil {
+
+	total := len(pkgs)
+	for idx, pkg := range pkgs {
+		stepNum := idx + 1
+		stepMsg := fmt.Sprintf("[%d/%d] %s", stepNum, total, pkg)
+		if err := nix.ProfileInstall(&nix.ProfileInstallArgs{
+			CustomStepMessage: stepMsg,
+			NixpkgsCommit:     d.cfg.Nixpkgs.Commit,
+			Package:           pkg,
+			ProfilePath:       profilePath,
+			Writer:            d.writer,
+		}); err != nil {
 			fmt.Fprintf(d.writer, "Error installing %s: %s", pkg, err)
 		} else {
 			fmt.Fprintf(d.writer, "%s is now installed\n", pkg)

internal/impl/packages.go

@@ -1,21 +1,17 @@
 package impl
 
 import (
-	"encoding/json"
 	"fmt"
-	"io/fs"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 
-	"github.com/fatih/color"
 	"github.com/pkg/errors"
 	"go.jetpack.io/devbox/internal/boxcli/featureflag"
 	"go.jetpack.io/devbox/internal/debug"
 	"go.jetpack.io/devbox/internal/fileutil"
 	"go.jetpack.io/devbox/internal/nix"
-	"go.jetpack.io/devbox/internal/xdg"
 )
 
 // packages.go has functions for adding, removing and getting info about nix packages
@@ -53,10 +49,6 @@ func (d *Devbox) addPackagesToProfile(mode installMode) error {
 		return nil
 	}
 
-	if err := d.ensureNixpkgsPrefetched(); err != nil {
-		return err
-	}
-
 	pkgs, err := d.pendingPackagesForInstallation()
 	if err != nil {
 		return err
@@ -84,29 +76,17 @@ func (d *Devbox) addPackagesToProfile(mode installMode) error {
 		stepNum := idx + 1
 
 		stepMsg := fmt.Sprintf("[%d/%d] %s", stepNum, total, pkg)
-		fmt.Printf("%s\n", stepMsg)
-
-		cmd := exec.Command(
-			"nix", "profile", "install",
-			"--profile", profileDir,
-			"--priority", d.getPackagePriority(pkg),
-			"--impure", // Needed to allow flags from environment to be used.
-			nix.FlakeNixpkgs(d.cfg.Nixpkgs.Commit)+"#"+pkg,
-		)
-		cmd.Args = append(cmd.Args, nix.ExperimentalFlags()...)
-		cmd.Stdout = &nixPackageInstallWriter{d.writer}
 
-		cmd.Env = nix.DefaultEnv()
-		cmd.Stderr = cmd.Stdout
-		err = cmd.Run()
-		if err != nil {
-			fmt.Fprintf(d.writer, "%s: ", stepMsg)
-			color.New(color.FgRed).Fprintf(d.writer, "Fail\n")
-			return errors.Wrapf(err, "Command: %s", cmd)
+		if err := nix.ProfileInstall(&nix.ProfileInstallArgs{
+			CustomStepMessage: stepMsg,
+			ExtraFlags:        []string{"--priority", d.getPackagePriority(pkg)},
+			NixpkgsCommit:     d.cfg.Nixpkgs.Commit,
+			Package:           pkg,
+			ProfilePath:       profileDir,
+			Writer:            d.writer,
+		}); err != nil {
+			return err
 		}
-
-		fmt.Fprintf(d.writer, "%s: ", stepMsg)
-		color.New(color.FgGreen).Fprintf(d.writer, "Success\n")
 	}
 
 	return nil
@@ -146,6 +126,7 @@ func (d *Devbox) removePackagesFromProfile(pkgs []string) error {
 			return errors.Errorf("Did not find AttributePath for package: %s", pkg)
 		}
 
+		// TODO: unify this with nix.ProfileRemove
 		cmd := exec.Command("nix", "profile", "remove",
 			"--profile", profileDir,
 			attrPath,
@@ -240,99 +221,3 @@ func resetProfileDirForFlakes(profileDir string) (err error) {
 
 	return errors.WithStack(os.Remove(profileDir))
 }
-
-// ensureNixpkgsPrefetched runs the prefetch step to download the flake of the registry
-func (d *Devbox) ensureNixpkgsPrefetched() error {
-	// Look up the cached map of commitHash:nixStoreLocation
-	commitToLocation, err := d.nixpkgsCommitFileContents()
-	if err != nil {
-		return err
-	}
-
-	// Check if this nixpkgs.Commit is located in the local /nix/store
-	location, isPresent := commitToLocation[d.cfg.Nixpkgs.Commit]
-	if isPresent {
-		if fi, err := os.Stat(location); err == nil && fi.IsDir() {
-			// The nixpkgs for this commit hash is present, so we don't need to prefetch
-			return nil
-		}
-	}
-
-	fmt.Fprintf(d.writer, "Ensuring nixpkgs registry is downloaded.\n")
-	cmd := exec.Command(
-		"nix", "flake", "prefetch",
-		nix.FlakeNixpkgs(d.cfg.Nixpkgs.Commit),
-	)
-	cmd.Args = append(cmd.Args, nix.ExperimentalFlags()...)
-	cmd.Env = nix.DefaultEnv()
-	cmd.Stdout = d.writer
-	cmd.Stderr = cmd.Stdout
-	if err := cmd.Run(); err != nil {
-		fmt.Fprintf(d.writer, "Ensuring nixpkgs registry is downloaded: ")
-		color.New(color.FgRed).Fprintf(d.writer, "Fail\n")
-		return errors.Wrapf(err, "Command: %s", cmd)
-	}
-	fmt.Fprintf(d.writer, "Ensuring nixpkgs registry is downloaded: ")
-	color.New(color.FgGreen).Fprintf(d.writer, "Success\n")
-
-	return d.saveToNixpkgsCommitFile(commitToLocation)
-}
-
-func (d *Devbox) nixpkgsCommitFileContents() (map[string]string, error) {
-	path := nixpkgsCommitFilePath()
-	if !fileutil.Exists(path) {
-		return map[string]string{}, nil
-	}
-
-	contents, err := os.ReadFile(path)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	commitToLocation := map[string]string{}
-	if err := json.Unmarshal(contents, &commitToLocation); err != nil {
-		return nil, errors.WithStack(err)
-	}
-	return commitToLocation, nil
-}
-
-func (d *Devbox) saveToNixpkgsCommitFile(commitToLocation map[string]string) error {
-	// Make a query to get the /nix/store path for this commit hash.
-	cmd := exec.Command("nix", "flake", "prefetch", "--json",
-		nix.FlakeNixpkgs(d.cfg.Nixpkgs.Commit),
-	)
-	cmd.Args = append(cmd.Args, nix.ExperimentalFlags()...)
-	out, err := cmd.Output()
-	if err != nil {
-		return errors.WithStack(err)
-	}
-
-	// read the json response
-	var prefetchData struct {
-		StorePath string `json:"storePath"`
-	}
-	if err := json.Unmarshal(out, &prefetchData); err != nil {
-		return errors.WithStack(err)
-	}
-
-	// Ensure the nixpkgs commit file path exists so we can write an update to it
-	path := nixpkgsCommitFilePath()
-	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil && !errors.Is(err, fs.ErrExist) {
-		return errors.WithStack(err)
-	}
-
-	// write to the map, jsonify it, and write that json to the nixpkgsCommit file
-	commitToLocation[d.cfg.Nixpkgs.Commit] = prefetchData.StorePath
-	serialized, err := json.Marshal(commitToLocation)
-	if err != nil {
-		return errors.WithStack(err)
-	}
-
-	err = os.WriteFile(path, serialized, 0644)
-	return errors.WithStack(err)
-}
-
-func nixpkgsCommitFilePath() string {
-	cacheDir := xdg.CacheSubpath("devbox")
-	return filepath.Join(cacheDir, "nixpkgs.json")
-}

internal/impl/util.go

@@ -17,12 +17,17 @@ const nixpkgsUtilityCommit = "f7475ce8950b761d80a13f3f81d2c23fce60c1dd"
 // It's used to install applications devbox might need, like process-compose
 // This is an alternative to a global install which would modify a user's
 // environment.
-func addDevboxUtilityPackage(pkg string) error {
+func (d *Devbox) addDevboxUtilityPackage(pkg string) error {
 	profilePath, err := utilityNixProfilePath()
 	if err != nil {
 		return err
 	}
-	return nix.ProfileInstall(profilePath, nixpkgsUtilityCommit, pkg)
+	return nix.ProfileInstall(&nix.ProfileInstallArgs{
+		NixpkgsCommit: nixpkgsUtilityCommit,
+		Package:       pkg,
+		ProfilePath:   profilePath,
+		Writer:        d.writer,
+	})
 }
 
 func utilityLookPath(binName string) (string, error) {

internal/nix/nixpkgs.go

@@ -0,0 +1,112 @@
+package nix
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	"go.jetpack.io/devbox/internal/fileutil"
+	"go.jetpack.io/devbox/internal/xdg"
+)
+
+// ensureNixpkgsPrefetched runs the prefetch step to download the flake of the registry
+func ensureNixpkgsPrefetched(w io.Writer, commit string) error {
+	// Look up the cached map of commitHash:nixStoreLocation
+	commitToLocation, err := nixpkgsCommitFileContents()
+	if err != nil {
+		return err
+	}
+
+	// Check if this nixpkgs.Commit is located in the local /nix/store
+	location, isPresent := commitToLocation[commit]
+	if isPresent {
+		if fi, err := os.Stat(location); err == nil && fi.IsDir() {
+			// The nixpkgs for this commit hash is present, so we don't need to prefetch
+			return nil
+		}
+	}
+
+	fmt.Fprintf(w, "Ensuring nixpkgs registry is downloaded.\n")
+	cmd := exec.Command(
+		"nix", "flake", "prefetch",
+		FlakeNixpkgs(commit),
+	)
+	cmd.Args = append(cmd.Args, ExperimentalFlags()...)
+	cmd.Env = DefaultEnv()
+	cmd.Stdout = w
+	cmd.Stderr = cmd.Stdout
+	if err := cmd.Run(); err != nil {
+		fmt.Fprintf(w, "Ensuring nixpkgs registry is downloaded: ")
+		color.New(color.FgRed).Fprintf(w, "Fail\n")
+		return errors.Wrapf(err, "Command: %s", cmd)
+	}
+	fmt.Fprintf(w, "Ensuring nixpkgs registry is downloaded: ")
+	color.New(color.FgGreen).Fprintf(w, "Success\n")
+
+	return saveToNixpkgsCommitFile(commit, commitToLocation)
+}
+
+func nixpkgsCommitFileContents() (map[string]string, error) {
+	path := nixpkgsCommitFilePath()
+	if !fileutil.Exists(path) {
+		return map[string]string{}, nil
+	}
+
+	contents, err := os.ReadFile(path)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	commitToLocation := map[string]string{}
+	if err := json.Unmarshal(contents, &commitToLocation); err != nil {
+		return nil, errors.WithStack(err)
+	}
+	return commitToLocation, nil
+}
+
+func saveToNixpkgsCommitFile(commit string, commitToLocation map[string]string) error {
+	// Make a query to get the /nix/store path for this commit hash.
+	cmd := exec.Command("nix", "flake", "prefetch", "--json",
+		FlakeNixpkgs(commit),
+	)
+	cmd.Args = append(cmd.Args, ExperimentalFlags()...)
+	out, err := cmd.Output()
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	// read the json response
+	var prefetchData struct {
+		StorePath string `json:"storePath"`
+	}
+	if err := json.Unmarshal(out, &prefetchData); err != nil {
+		return errors.WithStack(err)
+	}
+
+	// Ensure the nixpkgs commit file path exists so we can write an update to it
+	path := nixpkgsCommitFilePath()
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil && !errors.Is(err, fs.ErrExist) {
+		return errors.WithStack(err)
+	}
+
+	// write to the map, jsonify it, and write that json to the nixpkgsCommit file
+	commitToLocation[commit] = prefetchData.StorePath
+	serialized, err := json.Marshal(commitToLocation)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	err = os.WriteFile(path, serialized, 0644)
+	return errors.WithStack(err)
+}
+
+func nixpkgsCommitFilePath() string {
+	cacheDir := xdg.CacheSubpath("devbox")
+	return filepath.Join(cacheDir, "nixpkgs.json")
+}