[package outputs] generate buildInputs for package-outputs in flake (#1721)

## Summary

This PR reads the `outputs` field from the Config for a package.

For packages with non-default outputs in the config, we specify:
1. They are not to be taken from the binary cache
2. They are exempt from the `flakeInput.buildInputs()` that are
specified in the generated flake's devshell.buildInputs

Instead, we generate a [`symlinkJoin`
derivation](https://nixos.org/manual/nixpkgs/stable/#trivial-builder-symlinkJoin)
of the form:
```
            (pkgs.symlinkJoin {
              name = "prometheus-combined";
              paths = [
                # each output is printed here
                nixpkgs-fd04be-pkgs.prometheus.cli
                nixpkgs-fd04be-pkgs.prometheus.out
              ];
            })
```

This `prometheus-combined` derivation is useful so that when we derive
the nix-profile from the flake's buildInputs as in #1692, it is treated
as a single package. Without this, we could inline each output as a
distinct buildInput in the flake's devshell, but then the nix profile
would treat each output as its own package.

## How was it tested?


Added testscript unit-test. Also, ran the same steps inside the
testscript manually.

setup:
```
# add package with multiple outputs
devbox add prometheus --outputs cli,out

# add package for a specific nixpkgs commit hash (as a control)
devbox add github:nixos/nixpkgs/fd04bea4cbf76f86f244b9e2549fca066db8ddff#hello
```
then the generated flake is:
```
❯ cat .devbox/gen/flake/flake.nix
{
   description = "A devbox shell";

   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/fd04bea4cbf76f86f244b9e2549fca066db8ddff";
     nixpkgs-fd04be.url = "github:NixOS/nixpkgs/fd04bea4cbf76f86f244b9e2549fca066db8ddff";
     gh-nixos-nixpkgs-fd04bea4cbf76f86f244b9e2549fca066db8ddff.url = "github:NixOS/nixpkgs/fd04bea4cbf76f86f244b9e2549fca066db8ddff";
   };

   outputs = {
     self,
     nixpkgs,
     nixpkgs-fd04be,
     gh-nixos-nixpkgs-fd04bea4cbf76f86f244b9e2549fca066db8ddff,
   }:
      let
        pkgs = nixpkgs.legacyPackages.x86_64-darwin;
        nixpkgs-fd04be-pkgs = (import nixpkgs-fd04be {
          system = "x86_64-darwin";
          config.allowUnfree = true;
          config.permittedInsecurePackages = [
          ];
        });
        gh-nixos-nixpkgs-fd04bea4cbf76f86f244b9e2549fca066db8ddff-pkgs = (import gh-nixos-nixpkgs-fd04bea4cbf76f86f244b9e2549fca066db8ddff {
          system = "x86_64-darwin";
          config.allowUnfree = true;
          config.permittedInsecurePackages = [
          ];
        });
      in
      {
        devShells.x86_64-darwin.default = pkgs.mkShell {
          buildInputs = [
            (builtins.fetchClosure {
              fromStore = "https://cache.nixos.org";
              fromPath = "/nix/store/0djljz0g4s6f55xcnw7fpzcy7af7rxid-go-1.21.4";
              inputAddressed = true;
            })




            (pkgs.symlinkJoin {
              name = "prometheus-combined";
              paths = [
                nixpkgs-fd04be-pkgs.prometheus.cli
                nixpkgs-fd04be-pkgs.prometheus.out
              ];
            })
            gh-nixos-nixpkgs-fd04bea4cbf76f86f244b9e2549fca066db8ddff-pkgs.hello
          ];
        };
      };
 }
```

this leads to `$buildInputs` in a `devbox shell` to be:
```
❯ echo $buildInputs
/nix/store/0djljz0g4s6f55xcnw7fpzcy7af7rxid-go-1.21.4 /nix/store/9dn3rzv04x63n0kz2jwpgz82rdlsa56h-prometheus-combined /nix/store/4xy6iv0ph2v6w7n06cw5ra7cmyvignkm-hello-2.12.1
```

where the `prometheus-combined` derivation has the `cli` and `out`
outputs combined:
```
❯ ls -al /nix/store/9dn3rzv04x63n0kz2jwpgz82rdlsa56h-prometheus-combined/bin/
total 0
dr-xr-xr-x 4 root wheel 128 Dec 31  1969 .
dr-xr-xr-x 4 root wheel 128 Dec 31  1969 ..
lrwxr-xr-x 1 root wheel  76 Dec 31  1969 prometheus -> /nix/store/hizpsf2f5gc7l810328382xicjb9gc73-prometheus-2.48.1/bin/prometheus
lrwxr-xr-x 1 root wheel  78 Dec 31  1969 promtool -> /nix/store/8x6psdqn3945pbs0ww5wanmfhvvz2iyl-prometheus-2.48.1-cli/bin/promtool
```

Author: savil

internal/devpkg/narinfo_cache.go

@@ -27,6 +27,10 @@ func (p *Package) IsInBinaryCache() (bool, error) {
 	if p.PatchGlibc {
 		return false, nil
 	}
+	// Packages with non-default outputs are not to be taken from the binary cache.
+	if len(p.Outputs) > 0 {
+		return false, nil
+	}
 	if eligible, err := p.isEligibleForBinaryCache(); err != nil {
 		return false, err
 	} else if !eligible {

internal/devpkg/package.go

@@ -78,6 +78,10 @@ type Package struct {
 	//    example: github:nixos/nixpkgs/5233fd2ba76a3accb5aaa999c00509a11fd0793c#hello
 	Raw string
 
+	// Outputs is a list of outputs to build from the package's derivation.
+	// If empty, the default output is used.
+	Outputs []string
+
 	// PatchGlibc applies a function to the package's derivation that
 	// patches any ELF binaries to use the latest version of nixpkgs#glibc.
 	PatchGlibc bool
@@ -113,6 +117,7 @@ func PackagesFromConfig(config *devconfig.Config, l lock.Locker) []*Package {
 		pkg := newPackage(cfgPkg.VersionedName(), cfgPkg.IsEnabledOnPlatform(), l)
 		pkg.DisablePlugin = cfgPkg.DisablePlugin
 		pkg.PatchGlibc = cfgPkg.PatchGlibc && nix.SystemIsLinux()
+		pkg.Outputs = cfgPkg.Outputs
 		result = append(result, pkg)
 	}
 	return result
@@ -126,6 +131,7 @@ func PackageFromStringWithOptions(raw string, locker lock.Locker, opts devopt.Ad
 	pkg := PackageFromStringWithDefaults(raw, locker)
 	pkg.DisablePlugin = opts.DisablePlugin
 	pkg.PatchGlibc = opts.PatchGlibc
+	pkg.Outputs = opts.Outputs
 	return pkg
 }
 

internal/shellgen/flake_input.go

@@ -2,6 +2,7 @@ package shellgen
 
 import (
 	"context"
+	"errors"
 	"runtime/trace"
 	"slices"
 	"strings"
@@ -50,9 +51,54 @@ func (f *flakeInput) PkgImportName() string {
 	return f.Name + "-pkgs"
 }
 
+type SymlinkJoin struct {
+	Name  string
+	Paths []string
+}
+
+// BuildInputsForSymlinkJoin returns a list of SymlinkJoin objects that can be used
+// as the buildInput. Used for packages that have non-default outputs that need to
+// be combined into a single buildInput.
+func (f *flakeInput) BuildInputsForSymlinkJoin() ([]*SymlinkJoin, error) {
+	joins := []*SymlinkJoin{}
+	for _, pkg := range f.Packages {
+		// skip packages that have no non-default outputs
+		if len(pkg.Outputs) == 0 {
+			continue
+		}
+
+		attributePath, err := pkg.FullPackageAttributePath()
+		if err != nil {
+			return nil, err
+		}
+
+		if pkg.PatchGlibc {
+			return nil, errors.New("patch_glibc is not yet supported for packages with non-default outputs")
+		}
+		joins = append(joins, &SymlinkJoin{
+			Name: pkg.String() + "-combined",
+			Paths: lo.Map(pkg.Outputs, func(output string, _ int) string {
+				if !f.IsNixpkgs() {
+					return f.Name + "." + attributePath + "." + output
+				}
+				parts := strings.Split(attributePath, ".")
+				return f.PkgImportName() + "." + strings.Join(parts[2:], ".") + "." + output
+			}),
+		})
+	}
+	return joins, nil
+}
+
 func (f *flakeInput) BuildInputs() ([]string, error) {
 	var err error
-	attributePaths := lo.Map(f.Packages, func(pkg *devpkg.Package, _ int) string {
+
+	// Filter out packages that have non-default outputs
+	// These are handled in BuildInputsForSymlinkJoin
+	packages := lo.Filter(f.Packages, func(pkg *devpkg.Package, _ int) bool {
+		return len(pkg.Outputs) == 0
+	})
+
+	attributePaths := lo.Map(packages, func(pkg *devpkg.Package, _ int) string {
 		attributePath, attributePathErr := pkg.FullPackageAttributePath()
 		if attributePathErr != nil {
 			err = attributePathErr

internal/shellgen/generate.go

@@ -30,7 +30,7 @@ var tmplFS embed.FS
 // devbox.PrintEnv, which is the core function from which devbox shell/run/direnv
 // functionality is derived.
 func GenerateForPrintEnv(ctx context.Context, devbox devboxer) error {
-	defer trace.StartRegion(ctx, "generateShellFiles").End()
+	defer trace.StartRegion(ctx, "GenerateForPrintEnv").End()
 
 	plan, err := newFlakePlan(ctx, devbox)
 	if err != nil {

internal/shellgen/tmpl/flake_remove_nixpkgs.nix.tmpl

@@ -46,7 +46,17 @@
             })
             {{- end }}
             {{- end }}
-            {{- range .FlakeInputs }}
+            {{- range $_, $flakeInput := .FlakeInputs }}
+            {{- range .BuildInputsForSymlinkJoin }}
+            (pkgs.symlinkJoin {
+              name = "{{.Name}}";
+              paths = [
+                {{- range .Paths }}
+                {{.}}
+                {{- end }}
+              ];
+            })
+            {{- end }}
             {{- range .BuildInputs }}
             {{.}}
             {{- end }}

testscripts/add/add_outputs.test.txt

@@ -0,0 +1,32 @@
+# Testscript to add packages with non-default outputs
+
+exec devbox init
+
+# Add prometheus with default outputs. It will not have promtool.
+exec devbox add prometheus
+exec devbox run -- prometheus --version
+! exec devbox run -- promtool --version
+
+# Add prometheus with cli and out outputs. It will have promtool as well.
+exec devbox add prometheus --outputs cli,out
+json.superset devbox.json expected_devbox.json
+exec devbox run -- promtool --version
+exec devbox run -- prometheus --version
+
+
+
+-- devbox.json --
+{
+  "packages": [
+  ]
+}
+
+-- expected_devbox.json --
+{
+  "packages": {
+    "prometheus": {
+      "version": "latest",
+      "outputs": ["cli", "out"]
+    }
+  }
+}