[flakes] Assign higher priority to later packages to resolve conflicting packages error (#738)

## Summary

We want to re-try nix profile install with package conflicts by setting
a higher priority (lower priority number) for later packages.

## How was it tested?

using the github.com/jetpack-io/devbox-examples repo

before:
```
devbox-examples/development/ruby on  main [!?] on ☁️  (us-east-2) on ☁️  [email protected] 💫 initial project sync
❯ devbox shell
Ensuring packages are installed.

Installing 2 packages: bundler, ruby_3_1.

[1/2] bundler
[1/2] bundler: Success
[2/2] ruby_3_1
	error: files '/nix/store/spgr12gk13af8flz7akbs18fj4whqss2-bundler-2.4.5/bin/bundle' and '/nix/store/l4wmx8lfn6hlcfmbyhmksm024f8hixm1-ruby-3.1.2/bin/bundle' have the same priority 5; use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' or type 'nix profile install --help' if using 'nix profile' to find out howto change the priority of one of the conflicting packages (0 being the highest priority)
[2/2] ruby_3_1: Fail
Error: Command: /nix/var/nix/profiles/default/bin/nix profile install --profile /Users/savil/code/jetpack/devbox-examples/development/ruby/.devbox/nix/profile/default --impure nixpkgs/f80ac848e3d6f0c12c52758c0f25c10c97ca3b62#ruby_3_1 --extra-experimental-features ca-derivations --option experimental-features nix-command flakes --priority 5: exit status 1

Error: There was an internal error. Run with DEVBOX_DEBUG=1 for a detailed error message, and consider reporting it at https://github.com/jetpack-io/devbox/issues
```

Then I added `devbox global add zig` to test the sorting for global
packages first and then project packages later.

after:
```
❯ DEVBOX_DEBUG=0 devbox shell
Ensuring packages are installed.

Installing 3 packages: zig, bundler, ruby_3_1.

[1/3] zig
[1/3] zig: Success
[2/3] bundler
[2/3] bundler: Success
[3/3] ruby_3_1
[3/3] ruby_3_1: Success
Starting a devbox shell...
(devbox)
```

after with DEVBOX_DEBUG=1:
```
❯ DEVBOX_DEBUG=1 devbox shell
...
Installing 3 packages: zig, bundler, ruby_3_1.

[DEBUG] 2023/03/09 21:19:24 /Users/savil/code/jetpack/devbox/internal/impl/packages.go:246: ERROR: resetProfileDirForFlakes error: lstat /Users/savil/code/jetpack/devbox-examples/development/ruby/.devbox/nix/profile/default: no such file or directory
[1/3] zig
[1/3] zig: Success
[2/3] bundler
[2/3] bundler: Success
[3/3] ruby_3_1
[DEBUG] 2023/03/09 21:19:27 /Users/savil/code/jetpack/devbox/internal/nix/writer.go:39: Hiding output for user: error: files '/nix/store/spgr12gk13af8flz7akbs18fj4whqss2-bundler-2.4.5/bin/bundle' and '/nix/store/l4wmx8lfn6hlcfmbyhmksm024f8hixm1-ruby-3.1.2/bin/bundle' have the same priority 5; use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' or type 'nix profile install --help' if using 'nix profile' to find out how to change the priority of one of the conflicting packages (0 being the highest priority)
[DEBUG] 2023/03/09 21:19:27 /Users/savil/code/jetpack/devbox/internal/nix/profile.go:286: Re-trying nix profile install with priority 4 for package ruby_3_1
[3/3] ruby_3_1: Success
Starting a devbox shell...

```

Author: savil

internal/impl/packages.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	"github.com/fatih/color"
@@ -272,6 +273,17 @@ func (d *Devbox) addPackagesToProfile(mode installMode) error {
 		return nil
 	}
 
+	// Packages with higher priority number i.e. lower actual priority
+	// are to be installed first, so that any conflicts are resolved in favor
+	// of later packages (having lower priority number i.e. higher actual priority)
+	//
+	// We use stable sort so that users can manually change the order of packages
+	// in their configs, if they have particular opinions about which package should
+	// win any conflicts.
+	sort.SliceStable(pkgs, func(i, j int) bool {
+		return d.getPackagePriority(pkgs[i]) > d.getPackagePriority(pkgs[j])
+	})
+
 	var msg string
 	if len(pkgs) == 1 {
 		msg = fmt.Sprintf("Installing package: %s.", pkgs[0])
@@ -293,7 +305,7 @@ func (d *Devbox) addPackagesToProfile(mode installMode) error {
 
 		if err := nix.ProfileInstall(&nix.ProfileInstallArgs{
 			CustomStepMessage: stepMsg,
-			ExtraFlags:        []string{"--priority", d.getPackagePriority(pkg)},
+			Priority:          d.getPackagePriority(pkg),
 			NixpkgsCommit:     d.cfg.Nixpkgs.Commit,
 			Package:           pkg,
 			ProfilePath:       profileDir,

internal/nix/profile.go

@@ -12,6 +12,7 @@ import (
 	"github.com/fatih/color"
 	"github.com/pkg/errors"
 	"go.jetpack.io/devbox/internal/boxcli/featureflag"
+	"go.jetpack.io/devbox/internal/debug"
 )
 
 // ProfileListItems returns a list of the installed packages
@@ -186,6 +187,7 @@ func (item *NixProfileListItem) String() string {
 
 type ProfileInstallArgs struct {
 	CustomStepMessage string
+	Priority          string
 	ExtraFlags        []string
 	NixpkgsCommit     string
 	Package           string
@@ -206,32 +208,91 @@ func ProfileInstall(args *ProfileInstallArgs) error {
 		fmt.Fprintf(args.Writer, "%s\n", stepMsg)
 	}
 
-	cmd := exec.Command("nix", "profile", "install",
+	err := profileInstall(args)
+	if err != nil {
+		if errors.Is(err, ErrPackageNotFound) {
+			return err
+		}
+
+		fmt.Fprintf(args.Writer, "%s: ", stepMsg)
+		color.New(color.FgRed).Fprintf(args.Writer, "Fail\n")
+		return err
+	}
+
+	fmt.Fprintf(args.Writer, "%s: ", stepMsg)
+	color.New(color.FgGreen).Fprintf(args.Writer, "Success\n")
+
+	return nil
+}
+
+func profileInstall(args *ProfileInstallArgs) error {
+
+	cmd := exec.Command(
+		"nix", "profile", "install",
 		"--profile", args.ProfilePath,
 		"--impure", // Needed to allow flags from environment to be used.
 		FlakeNixpkgs(args.NixpkgsCommit)+"#"+args.Package,
 	)
 	cmd.Args = append(cmd.Args, ExperimentalFlags()...)
+	if args.Priority != "" {
+		cmd.Args = append(cmd.Args, "--priority", args.Priority)
+	}
 	cmd.Args = append(cmd.Args, args.ExtraFlags...)
 
 	cmd.Env = DefaultEnv()
-	cmd.Stdout = &PackageInstallWriter{args.Writer}
-	var stderr bytes.Buffer
-	cmd.Stderr = io.MultiWriter(&stderr, cmd.Stdout)
+	writer := &PackageInstallWriter{args.Writer}
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = io.MultiWriter(&stdout, writer)
+	cmd.Stderr = io.MultiWriter(&stderr, writer)
 
 	if err := cmd.Run(); err != nil {
 		if strings.Contains(stderr.String(), "does not provide attribute") {
 			return ErrPackageNotFound
 		}
 
-		fmt.Fprintf(args.Writer, "%s: ", stepMsg)
-		color.New(color.FgRed).Fprintf(args.Writer, "Fail\n")
+		// If two packages being installed seek to install two nix packages (could be themselves,
+		// or could be a dependency) that have the same binary name,
+		// then `nix profile install` will fail with `conflicting packages` error (as of nix version 2.14.0)
+		//
+		// An example error message looks like the following:
+		// error: files '/nix/store/spgr12gk13af8flz7akbs18fj4whqss2-bundler-2.4.5/bin/bundle' and
+		// '/nix/store/l4wmx8lfn6hlcfmbyhmksm024f8hixm1-ruby-3.1.2/bin/bundle' have the same priority 5;
+		// use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' or type 'nix profile install --help'
+		// if using 'nix profile' to find out howto change the priority of one of the conflicting packages
+		// (0 being the highest priority)
+		//
+		// However, for the purposes of starting a shell with these packages, nix flakes will give
+		// precedence to the later package. We enable similar functionality by increasing the priority
+		// of any package being installed and conflicting with a previously installed package: the
+		// package being installed later "wins".
+		isConflictingPackagesError := strings.Contains(stdout.String(), "conflicting packages") ||
+			strings.Contains(stderr.String(), "conflicting packages")
+		if isConflictingPackagesError && args.Priority != "0" {
+
+			priority := args.Priority
+			if priority == "" {
+				priority = "5" // 5 is the default priority in `nix profile`
+			}
+
+			intPriority, strconvErr := strconv.Atoi(priority)
+			if strconvErr != nil {
+				debug.Log(
+					"Error: falling back to regular error handling logic due to strconv.Atoi error: %s",
+					strconvErr)
+				// fallthrough to the regular error handling logic
+			} else {
+				// to give higher priority, we need to assign a lower priority number
+				args.Priority = strconv.Itoa(intPriority - 1)
+				debug.Log("Re-trying nix profile install with priority %s for package %s\n",
+					args.Priority,
+					args.Package,
+				)
+				return profileInstall(args)
+			}
+		}
+
 		return errors.Wrapf(err, "Command: %s", cmd)
 	}
-
-	fmt.Fprintf(args.Writer, "%s: ", stepMsg)
-	color.New(color.FgGreen).Fprintf(args.Writer, "Success\n")
-
 	return nil
 }
 

internal/nix/writer.go

@@ -3,11 +3,16 @@ package nix
 import (
 	"io"
 	"strings"
+
+	"go.jetpack.io/devbox/internal/debug"
 )
 
-var packageInstallIgnore = []string{
-	`replacing old 'devbox-development'`,
-	`installing 'devbox-development'`,
+// packageInstallIgnore will skip lines that have the strings in the keys of this map.
+// The boolean values inform the writer whether to log the line to debug.Log.
+var packageInstallIgnore = map[string]bool{
+	`replacing old 'devbox-development'`: false,
+	`installing 'devbox-development'`:    false,
+	`conflicting packages`:               true,
 }
 
 type PackageInstallWriter struct {
@@ -28,8 +33,11 @@ func (fw *PackageInstallWriter) Write(p []byte) (n int, err error) {
 }
 
 func (*PackageInstallWriter) ignore(line string) bool {
-	for _, filter := range packageInstallIgnore {
+	for filter, shouldLog := range packageInstallIgnore {
 		if strings.Contains(line, filter) {
+			if shouldLog {
+				debug.Log("Hiding output for user: %s", line)
+			}
 			return true
 		}
 	}