[perf] Improve how we determine attribute path (#1050)

## Summary

This makes 2 improvements to our use of attribute paths which was
causing performance issues:

**Improve ProfileListIndex:**

ProfileListIndex will attempt to find an exact match in unlocked profile
references. (See `normalizedDevboxPackageReference`. This short circuits
having to loop through all profile list items and having to compute
normalized attribute paths for each.

**Split PackageAttributePath:**

Split `PackageAttributePath` into `NormalizedPackageAttributePath` and a
faster new `PackageAttributePath`. When computing attribute paths we
would always use nix search to ensure the attribute path is normalized.
This is helpful for comparing 2 distinct attribute paths that point to
same package (e.g. `flake` and `flake#default` are likely the same). The
downside is that search is slow. (about 100ms per search). So what we do
is when using devbox packages, we assume they resolve to nixpkgs and
therefore we can reconstruct the exact attribute path
(`legacyPackages.<system>.<pkg>`) without using search.

Some additional improvements:

* Currently local/remote flakes still need to use search. This affects
our plugins. We could cache the search results to make them faster.

## How was it tested?

Created devbox.json with 20 packages and stuff got super slow. After
these changes, it reduces non-installation time during `devbox add` from
40-60 seconds to about 2s.

Author: mikeland73

internal/impl/packages.go

@@ -277,15 +277,19 @@ func (d *Devbox) removePackagesFromProfile(ctx context.Context, pkgs []string) e
 		return err
 	}
 
-	for _, pkg := range pkgs {
+	for _, input := range nix.InputsFromStrings(pkgs, d.lockfile) {
 		index, err := nix.ProfileListIndex(&nix.ProfileListIndexArgs{
 			Lockfile:   d.lockfile,
 			Writer:     d.writer,
-			Pkg:        pkg,
+			Input:      input,
 			ProfileDir: profileDir,
 		})
 		if err != nil {
-			ux.Ferror(d.writer, "Package %s not found in profile. Skipping.\n", pkg)
+			ux.Ferror(
+				d.writer,
+				"Package %s not found in profile. Skipping.\n",
+				input.Raw,
+			)
 			continue
 		}
 
@@ -322,16 +326,16 @@ func (d *Devbox) pendingPackagesForInstallation(ctx context.Context) ([]string,
 	if err != nil {
 		return nil, err
 	}
-	for _, pkg := range d.Packages() {
+	for _, input := range d.packagesAsInputs() {
 		_, err := nix.ProfileListIndex(&nix.ProfileListIndexArgs{
 			List:       list,
 			Lockfile:   d.lockfile,
 			Writer:     d.writer,
-			Pkg:        pkg,
+			Input:      input,
 			ProfileDir: profileDir,
 		})
 		if err != nil {
-			pending = append(pending, pkg)
+			pending = append(pending, input.Raw)
 		}
 	}
 	return pending, nil

internal/nix/input.go

@@ -19,7 +19,6 @@ import (
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/cuecfg"
 	"go.jetpack.io/devbox/internal/lock"
-	"go.jetpack.io/devbox/internal/searcher"
 )
 
 type Input struct {
@@ -108,10 +107,54 @@ func (i *Input) URLForInstall() (string, error) {
 	return i.urlWithoutFragment() + "#" + attrPath, nil
 }
 
-// PackageAttributePath returns just the name for non-flakes. For flake
-// references is returns the full path to the package in the flake. e.g.
-// packages.x86_64-linux.hello
+func (i *Input) normalizedDevboxPackageReference() (string, error) {
+	if !i.IsDevboxPackage() {
+		return "", nil
+	}
+
+	path := ""
+	if i.isVersioned() {
+		entry, err := i.lockfile.Resolve(i.String())
+		if err != nil {
+			return "", err
+		}
+		path = entry.Resolved
+	} else if i.IsDevboxPackage() {
+		path = i.lockfile.LegacyNixpkgsPath(i.String())
+	}
+
+	if path != "" {
+		s, err := System()
+		if err != nil {
+			return "", err
+		}
+		url, fragment, _ := strings.Cut(path, "#")
+		return fmt.Sprintf("%s#legacyPackages.%s.%s", url, s, fragment), nil
+	}
+
+	return "", nil
+}
+
+// PackageAttributePath returns the attribute path for a package. It is not
+// always normalized which means it should not be used to compare packages.
+// During happy paths (devbox packages and nix flakes that contains a fragment)
+// it is much faster than NormalizedPackageAttributePath
 func (i *Input) PackageAttributePath() (string, error) {
+	if i.IsDevboxPackage() {
+		reference, err := i.normalizedDevboxPackageReference()
+		if err != nil {
+			return "", err
+		}
+		_, fragment, _ := strings.Cut(reference, "#")
+		return fragment, nil
+	}
+	return i.NormalizedPackageAttributePath()
+}
+
+// NormalizedPackageAttributePath returns an attribute path normalized by nix
+// search. This is useful for comparing different attribute paths that may
+// point to the same package. Note, it's an expensive call.
+func (i *Input) NormalizedPackageAttributePath() (string, error) {
 	var query string
 	if i.isVersioned() {
 		entry, err := i.lockfile.Resolve(i.String())
@@ -195,14 +238,10 @@ func (i *Input) hash() string {
 }
 
 func (i *Input) ValidateExists() (bool, error) {
-	if i.isVersioned() {
-		version := i.version()
-		if version == "" && i.isVersioned() {
-			return false, usererr.New("No version specified for %q.", i.Path)
-		}
-		return searcher.Exists(i.CanonicalName(), version)
+	if i.isVersioned() && i.version() == "" {
+		return false, usererr.New("No version specified for %q.", i.Path)
 	}
-	info, err := i.PackageAttributePath()
+	info, err := i.NormalizedPackageAttributePath()
 	return info != "", err
 }
 
@@ -216,11 +255,11 @@ func (i *Input) equals(other *Input) bool {
 		return false
 	}
 
-	name, err := i.PackageAttributePath()
+	name, err := i.NormalizedPackageAttributePath()
 	if err != nil {
 		return false
 	}
-	otherName, err := other.PackageAttributePath()
+	otherName, err := other.NormalizedPackageAttributePath()
 	if err != nil {
 		return false
 	}

internal/nix/profile.go

@@ -25,10 +25,11 @@ import (
 
 const DefaultPriority = 5
 
-type NixProfileList []*NixProfileListItem
-
 // ProfileListItems returns a list of the installed packages
-func ProfileListItems(writer io.Writer, profileDir string) (NixProfileList, error) {
+func ProfileListItems(
+	writer io.Writer,
+	profileDir string,
+) (map[string]*NixProfileListItem, error) {
 	cmd := exec.Command(
 		"nix", "profile", "list",
 		"--profile", profileDir,
@@ -55,7 +56,7 @@ func ProfileListItems(writer io.Writer, profileDir string) (NixProfileList, erro
 		return nil, redact.Errorf("error starting \"nix profile list\" command: %w", err)
 	}
 
-	items := []*NixProfileListItem{}
+	items := map[string]*NixProfileListItem{}
 	scanner := bufio.NewScanner(out)
 	scanner.Split(bufio.ScanLines)
 
@@ -66,7 +67,7 @@ func ProfileListItems(writer io.Writer, profileDir string) (NixProfileList, erro
 			return nil, err
 		}
 
-		items = append(items, item)
+		items[item.unlockedReference] = item
 	}
 
 	if err := cmd.Wait(); err != nil {
@@ -78,10 +79,10 @@ func ProfileListItems(writer io.Writer, profileDir string) (NixProfileList, erro
 type ProfileListIndexArgs struct {
 	// For performance you can reuse the same list in multiple operations if you
 	// are confident index has not changed.
-	List       NixProfileList
+	List       map[string]*NixProfileListItem
 	Lockfile   *lock.File
 	Writer     io.Writer
-	Pkg        string
+	Input      *Input
 	ProfileDir string
 }
 
@@ -95,12 +96,20 @@ func ProfileListIndex(args *ProfileListIndexArgs) (int, error) {
 		}
 	}
 
-	input := InputFromString(args.Pkg, args.Lockfile)
+	// This is an optimization for happy path. A resolved devbox package
+	// should match the unlockedReference of an existing profile item.
+	ref, err := args.Input.normalizedDevboxPackageReference()
+	if err != nil {
+		return -1, err
+	}
+	if item, found := list[ref]; found {
+		return item.index, nil
+	}
 
 	for _, item := range list {
 		existing := InputFromString(item.unlockedReference, args.Lockfile)
 
-		if input.equals(existing) {
+		if args.Input.equals(existing) {
 			return item.index, nil
 		}
 	}