[devbox add] --allow-insecure should handle multiple, user-specified packages (#1749)

## Summary

Changes
1. Modify `devbox add`'s `--allow-insecure` flag to take a `[]string`
instead of `bool`.
2. Save `allow_insecure` field in the package in the `devbox.json`
config
3. Set `NIXPKGS_ALLOW_INSECURE=1` in the `nix build` during `devbox add`
4. Set the packages specified in `allow_insecure` in the generated
flake's `permittedInsecurePackages`.

TODO:
- [x] Handle existing projects with `allow_insecure: true` in their
`devbox.lock` (#1754)

**Downside:**
If a user erroneously specifies the wrong package name or names in
`--allow-insecure=<packages>`, then a `devbox add <pkg>
--allow-insecure=<packages>` outside the `devbox shell` environment will
quietly work. However, the user will experience an error when running
`devbox shell` (or similar) next time, and need to fix the erroneous
`--allow-insecure` value.
- Fix: we could fix this by doing "recomputeState" when
installing-with-insecure in "ensureStateIsUpToDate", but I haven't to
avoid adding complexity for this unlikely scenario. I call it unlikely
because the error message (see below) specifically tells the user what
values to use with `--allow-insecure=<packages>`

Fixes #1337 

## How was it tested?

`devbox add [email protected]` gives the following error now:
```
 devbox add [email protected]
Info: Adding package "[email protected]" to devbox.json
[1/1] python2
[1/1] python2: Fail

Error: Nix: Package  ‘python-2.7.18.7’ is insecure.

Known vulnerabilities:
Python 2.7 has reached its end of life after 2020-01-01. See https://www.python.org/doc/sunset-python-2/.

To override, use `devbox add <pkg> --allow-insecure=python-2.7.18.7`
```

Doing `devbox add [email protected] --allow-insecure=python-2.7.18.1` works
with the devbox.json now having:
```
+    "python": {
+      "version":        "2.7",
+      "allow_insecure": ["python-2.7.18.7"],
+    },
```


Also for the package reported in #1337:


<img width="907" alt="Screenshot 2024-01-25 at 5 43 43 PM"
src="https://github.com/jetpack-io/devbox/assets/676452/8d922d43-0c1c-4e86-ae6e-12fe9bc9e82d">

Author: savil

internal/boxcli/add.go

@@ -19,7 +19,7 @@ const toSearchForPackages = "To search for packages, use the `devbox search` com
 
 type addCmdFlags struct {
 	config           configFlags
-	allowInsecure    bool
+	allowInsecure    []string
 	disablePlugin    bool
 	platforms        []string
 	excludePlatforms []string
@@ -53,8 +53,8 @@ func addCmd() *cobra.Command {
 	}
 
 	flags.config.register(command)
-	command.Flags().BoolVar(
-		&flags.allowInsecure, "allow-insecure", false,
+	command.Flags().StringSliceVar(
+		&flags.allowInsecure, "allow-insecure", []string{},
 		"allow adding packages marked as insecure.")
 	command.Flags().BoolVar(
 		&flags.disablePlugin, "disable-plugin", false,

internal/devbox/devopt/devboxopts.go

@@ -43,7 +43,7 @@ type Credentials struct {
 }
 
 type AddOpts struct {
-	AllowInsecure    bool
+	AllowInsecure    []string
 	Platforms        []string
 	ExcludePlatforms []string
 	DisablePlugin    bool

internal/devbox/packages.go

@@ -137,24 +137,9 @@ func (d *Devbox) setPackageOptions(pkgs []string, opts devopt.AddOpts) error {
 			d.stderr, pkg, opts.Outputs); err != nil {
 			return err
 		}
-	}
-
-	// Resolving here ensures we allow insecure before running ensureStateIsUpToDate
-	// which will call print-dev-env. Resolving does not save the lockfile, we
-	// save at the end when everything has succeeded.
-	if opts.AllowInsecure {
-		nix.AllowInsecurePackages()
-		for _, name := range pkgs {
-			p, err := d.lockfile.Resolve(name)
-			if err != nil {
-				return err
-			}
-			// TODO: Now that config packages can have fields,
-			// we should set this in the config, not the lockfile.
-			if !p.AllowInsecure {
-				fmt.Fprintf(d.stderr, "Allowing insecure for %s\n", name)
-			}
-			p.AllowInsecure = true
+		if err := d.cfg.Packages.SetAllowInsecure(
+			d.stderr, pkg, opts.AllowInsecure); err != nil {
+			return err
 		}
 	}
 
@@ -179,7 +164,7 @@ func (d *Devbox) printPostAddMessage(
 		}
 	}
 
-	if len(opts.Platforms) == 0 && len(opts.ExcludePlatforms) == 0 && len(opts.Outputs) == 0 && !opts.AllowInsecure {
+	if len(opts.Platforms) == 0 && len(opts.ExcludePlatforms) == 0 && len(opts.Outputs) == 0 && len(opts.AllowInsecure) == 0 {
 		if len(unchangedPackageNames) == 1 {
 			ux.Finfo(d.stderr, "Package %q was already in devbox.json and was not modified\n", unchangedPackageNames[0])
 		} else if len(unchangedPackageNames) > 1 {
@@ -444,8 +429,12 @@ func (d *Devbox) installNixPackagesToStore(ctx context.Context) error {
 		stepMsg := fmt.Sprintf("[%d/%d] %s", stepNum, total, pkg)
 		fmt.Fprintf(d.stderr, stepMsg+"\n")
 
-		// --no-link to avoid generating the result objects
-		err = nix.Build(ctx, []string{"--no-link"}, installable)
+		args := &nix.BuildArgs{
+			AllowInsecure: pkg.HasAllowInsecure(),
+			// --no-link to avoid generating the result objects
+			Flags: []string{"--no-link"},
+		}
+		err = nix.Build(ctx, args, installable)
 		if err != nil {
 			fmt.Fprintf(d.stderr, "%s: ", stepMsg)
 			color.New(color.FgRed).Fprintf(d.stderr, "Fail\n")
@@ -475,6 +464,10 @@ func (d *Devbox) installNixPackagesToStore(ctx context.Context) error {
 				)
 			}
 
+			if isInsecureErr, userErr := nix.IsExitErrorInsecurePackage(err, installable); isInsecureErr {
+				return userErr
+			}
+
 			return usererr.WithUserMessage(err, "error installing package %s", pkg.Raw)
 		}
 

internal/devconfig/ast.go

@@ -208,6 +208,14 @@ func (c *configAST) appendOutputs(name, fieldName string, outputs []string) {
 	c.appendStringSliceField(name, fieldName, outputs)
 }
 
+func (c *configAST) appendAllowInsecure(name, fieldName string, whitelist []string) {
+	if len(whitelist) == 0 {
+		return
+	}
+
+	c.appendStringSliceField(name, fieldName, whitelist)
+}
+
 func (c *configAST) FindPkgObject(name string) *hujson.Object {
 	pkgs := c.packagesField(true).Value.Value.(*hujson.Object)
 	i := c.memberIndex(pkgs, name)

internal/devconfig/config_test.go

@@ -653,6 +653,38 @@ func TestSetOutputsMigrateArray(t *testing.T) {
 	}
 }
 
+func TestSetAllowInsecure(t *testing.T) {
+	in, want := parseConfigTxtarTest(t, `
+-- in --
+{
+  "packages": {
+    "python": {
+      "version": "2.7"
+    }
+  }
+}
+-- want --
+{
+  "packages": {
+    "python": {
+      "version":        "2.7",
+      "allow_insecure": ["python-2.7.18.1"]
+    }
+  }
+}`)
+
+	err := in.Packages.SetAllowInsecure(io.Discard, "[email protected]", []string{"python-2.7.18.1"})
+	if err != nil {
+		t.Error(err)
+	}
+	if diff := cmp.Diff(want, in.Bytes(), optParseHujson()); diff != "" {
+		t.Errorf("wrong parsed config json (-want +got):\n%s", diff)
+	}
+	if diff := cmp.Diff(want, in.Bytes()); diff != "" {
+		t.Errorf("wrong raw config hujson (-want +got):\n%s", diff)
+	}
+}
+
 func TestDefault(t *testing.T) {
 	path := filepath.Join(t.TempDir())
 	in := DefaultConfig()

internal/devconfig/packages.go

@@ -225,6 +225,29 @@ func (pkgs *Packages) SetOutputs(writer io.Writer, versionedName string, outputs
 	return nil
 }
 
+func (pkgs *Packages) SetAllowInsecure(writer io.Writer, versionedName string, whitelist []string) error {
+	name, version := parseVersionedName(versionedName)
+	i := pkgs.index(name, version)
+	if i == -1 {
+		return errors.Errorf("package %s not found", versionedName)
+	}
+
+	toAdd := []string{}
+	for _, w := range whitelist {
+		if !slices.Contains(pkgs.Collection[i].AllowInsecure, w) {
+			toAdd = append(toAdd, w)
+		}
+	}
+
+	if len(toAdd) > 0 {
+		pkg := &pkgs.Collection[i]
+		pkgs.ast.appendAllowInsecure(pkg.name, "allow_insecure", toAdd)
+		pkg.AllowInsecure = append(pkg.AllowInsecure, toAdd...)
+		ux.Finfo(writer, "Allowed insecure %s for package %s\n", strings.Join(toAdd, ", "), versionedName)
+	}
+	return nil
+}
+
 func (pkgs *Packages) index(name, version string) int {
 	return slices.IndexFunc(pkgs.Collection, func(p Package) bool {
 		return p.name == name && p.Version == version
@@ -246,6 +269,10 @@ type Package struct {
 	// Outputs is the list of outputs to use for this package, assuming
 	// it is a nix package. If empty, the default output is used.
 	Outputs []string `json:"outputs,omitempty"`
+
+	// AllowInsecure is a whitelist of packages that may be marked insecure
+	// in nixpkgs, but are allowed by the user to be installed.
+	AllowInsecure []string `json:"allow_insecure,omitempty"`
 }
 
 func NewVersionOnlyPackage(name, version string) Package {
@@ -276,13 +303,18 @@ func NewPackage(name string, values map[string]any) Package {
 	if o, ok := values["outputs"]; ok {
 		outputs = o.([]string)
 	}
+	var allowInsecure []string
+	if a, ok := values["allow_insecure"]; ok {
+		allowInsecure = a.([]string)
+	}
 
 	return Package{
 		name:              name,
 		Version:           version.(string),
 		Platforms:         platforms,
 		ExcludedPlatforms: excludedPlatforms,
 		Outputs:           outputs,
+		AllowInsecure:     allowInsecure,
 	}
 }
 

internal/devconfig/packages_test.go

@@ -174,6 +174,22 @@ func TestJsonifyConfigPackages(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "map-with-allow-insecure-nixpkgs-reference",
+			jsonConfig: `{"packages":{"github:nixos/nixpkgs/5233fd2ba76a3accb5aaa999c00509a11fd0793c#python":` +
+				`{"version":"2.7",` +
+				`"allow_insecure":["python-2.7.18.1"]` +
+				`}}}`,
+
+			expected: Packages{
+				Collection: []Package{
+					NewPackage("github:nixos/nixpkgs/5233fd2ba76a3accb5aaa999c00509a11fd0793c#python", map[string]any{
+						"version":        "2.7",
+						"allow_insecure": []string{"python-2.7.18.1"},
+					}),
+				},
+			},
+		},
 	}
 
 	for _, testCase := range testCases {

internal/devpkg/package.go

@@ -86,6 +86,10 @@ type Package struct {
 	// patches any ELF binaries to use the latest version of nixpkgs#glibc.
 	PatchGlibc bool
 
+	// AllowInsecure are a list of nix packages that are whitelisted to be
+	// installed even if they are marked as insecure.
+	AllowInsecure []string
+
 	// isInstallable is true if the package may be enabled on the current platform.
 	isInstallable bool
 
@@ -118,6 +122,7 @@ func PackagesFromConfig(config *devconfig.Config, l lock.Locker) []*Package {
 		pkg.DisablePlugin = cfgPkg.DisablePlugin
 		pkg.PatchGlibc = cfgPkg.PatchGlibc && nix.SystemIsLinux()
 		pkg.Outputs = cfgPkg.Outputs
+		pkg.AllowInsecure = cfgPkg.AllowInsecure
 		result = append(result, pkg)
 	}
 	return result
@@ -132,6 +137,7 @@ func PackageFromStringWithOptions(raw string, locker lock.Locker, opts devopt.Ad
 	pkg.DisablePlugin = opts.DisablePlugin
 	pkg.PatchGlibc = opts.PatchGlibc
 	pkg.Outputs = opts.Outputs
+	pkg.AllowInsecure = opts.AllowInsecure
 	return pkg
 }
 
@@ -572,8 +578,8 @@ func (p *Package) InputAddressedPath() (string, error) {
 	return sysInfo.StorePath, nil
 }
 
-func (p *Package) AllowInsecure() bool {
-	return p.lockfile.Get(p.Raw).IsAllowInsecure()
+func (p *Package) HasAllowInsecure() bool {
+	return len(p.AllowInsecure) > 0
 }
 
 // StoreName returns the last section of the store path. Example:

internal/nix/build.go

@@ -10,12 +10,21 @@ import (
 	"go.jetpack.io/devbox/internal/debug"
 )
 
-func Build(ctx context.Context, flags []string, installables ...string) error {
-	// --impure is required for allowUnfreeEnv to work.
+type BuildArgs struct {
+	AllowInsecure bool
+	Flags         []string
+}
+
+func Build(ctx context.Context, args *BuildArgs, installables ...string) error {
+	// --impure is required for allowUnfreeEnv/allowInsecureEnv to work.
 	cmd := commandContext(ctx, "build", "--impure")
-	cmd.Args = append(cmd.Args, flags...)
+	cmd.Args = append(cmd.Args, args.Flags...)
 	cmd.Args = append(cmd.Args, installables...)
 	cmd.Env = allowUnfreeEnv(os.Environ())
+	if args.AllowInsecure {
+		debug.Log("Setting Allow-insecure env-var\n")
+		cmd.Env = allowInsecureEnv(cmd.Env)
+	}
 
 	debug.Log("Running cmd: %s\n", cmd)
 	_, err := cmd.Output()

internal/nix/nix.go

@@ -6,6 +6,7 @@ package nix
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io/fs"
 	"os"
 	"os/exec"
@@ -72,7 +73,7 @@ func (*Nix) PrintDevEnv(ctx context.Context, args *PrintDevEnvArgs) (*PrintDevEn
 		cmd.Args = append(cmd.Args, "--json")
 		debug.Log("Running print-dev-env cmd: %s\n", cmd)
 		data, err = cmd.Output()
-		if insecure, insecureErr := isExitErrorInsecurePackage(err); insecure {
+		if insecure, insecureErr := IsExitErrorInsecurePackage(err, "" /*installable*/); insecure {
 			return nil, insecureErr
 		} else if err != nil {
 			return nil, errors.Wrapf(err, "Command: %s", cmd)
@@ -223,18 +224,58 @@ func ProfileBinPath(projectDir string) string {
 	return filepath.Join(projectDir, ProfilePath, "bin")
 }
 
-func isExitErrorInsecurePackage(err error) (bool, error) {
+func IsExitErrorInsecurePackage(err error, installableOrEmpty string) (bool, error) {
 	var exitErr *exec.ExitError
 	if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 {
 		if strings.Contains(string(exitErr.Stderr), "is marked as insecure") {
-			re := regexp.MustCompile(`Package ([^ ]+)`)
-			match := re.FindStringSubmatch(string(exitErr.Stderr))
-			return true, usererr.New(
-				"Package %s is insecure. \n\n"+
-					"To override use `devbox add <pkg> --allow-insecure`",
-				match[0],
-			)
+			packageRegex := regexp.MustCompile(`Package ([^ ]+)`)
+			packageMatch := packageRegex.FindStringSubmatch(string(exitErr.Stderr))
+
+			knownVulnerabilities := []string{}
+			if installableOrEmpty != "" {
+				knownVulnerabilities = PackageKnownVulnerabilities(installableOrEmpty)
+			}
+
+			insecurePackages := parseInsecurePackagesFromExitError(string(exitErr.Stderr))
+
+			// Construct the error message.
+			errMessages := []string{}
+			errMessages = append(errMessages, fmt.Sprintf("Package %s is insecure.", packageMatch[1]))
+			if len(knownVulnerabilities) > 0 {
+				errMessages = append(errMessages,
+					fmt.Sprintf("Known vulnerabilities:\n%s", strings.Join(knownVulnerabilities, "\n")))
+			}
+			errMessages = append(errMessages,
+				fmt.Sprintf("To override, use `devbox add <pkg> --allow-insecure=%s`", strings.Join(insecurePackages, ", ")))
+
+			return true, usererr.New(strings.Join(errMessages, "\n\n"))
 		}
 	}
 	return false, nil
 }
+
+func parseInsecurePackagesFromExitError(errorMsg string) []string {
+	insecurePackages := []string{}
+
+	// permittedRegex is designed to match the following:
+	// permittedInsecurePackages = [
+	//    "package-one"
+	//    "package-two"
+	// ];
+	permittedRegex := regexp.MustCompile(`permittedInsecurePackages\s*=\s*\[([\s\S]*?)\]`)
+	permittedMatch := permittedRegex.FindStringSubmatch(errorMsg)
+	if len(permittedMatch) > 1 {
+		packagesList := permittedMatch[1]
+		// pick out the package name strings inside the quotes
+		packageMatches := regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(packagesList, -1)
+
+		// Extract the insecure package names from the matches
+		for _, packageMatch := range packageMatches {
+			if len(packageMatch) > 1 {
+				insecurePackages = append(insecurePackages, packageMatch[1])
+			}
+		}
+	}
+
+	return insecurePackages
+}