[lockfile] Use lockfile for legacy packages (#964)

## Summary

This changes saves legacy packages to lockfile. It does not save
absolute flake references to lockfile (e.g. `github:/my/flake#foo`) but
we could in the future.

Merging legacy and versioned packages let's us simplify the code quite a
bit. Specifically now all inputs are treated like flakes (either
explicit or resolved).

One complication this exposes is the php and haskell planners.
Specifically, each planner creates definitions must be included as
packages. As a quick hack I just include the definitions as is at the
top of the list. This ensures these take precedence over any other
package. It also means that packages can show up twice if these planners
are activated. That doesn't break the functionality, it just looks weird
in the flake.nix

@Lagoja are there any example that exercises the Haskell planner?

A few possible follow ups:

- [ ] We only create lockfile when packages need to be resolved. This
may lead to some perf issues is people use an existing project on a read
only platform (e.g. CICD) because not having a lockfile makes things a
bit slower.
- [ ] Out flake.nix is a bit ugly because now it has multiple duplicate
inputs. We can clean this up.

## How was it tested?

Need to do a lot more testing but what I did:

```
devbox rm go_1_20 && devbox add go_1_20
rm -rf .devbox
devbox install 
devbox run echo 5
devbox add [email protected]
devbox run "python --version"
devbox shell
```

Need to test:

- [x] LOCKFILE flag off on legacy project
- [x] LOCKFILE flag on on legacy project
- [x] LOCKFILE flag off on project with lockfile
- [x] LOCKFILE flag on on project with lockfile

Author: mikeland73

devbox.lock

@@ -0,0 +1,20 @@
+{
+  "lockfile_version": "1",
+  "packages": {
+    "actionlint": {
+      "last_modified": "",
+      "resolved": "github:NixOS/nixpkgs/3364b5b117f65fe1ce65a3cdd5612a078a3b31e3#actionlint",
+      "version": ""
+    },
+    "go_1_20": {
+      "last_modified": "",
+      "resolved": "github:NixOS/nixpkgs/3364b5b117f65fe1ce65a3cdd5612a078a3b31e3#go_1_20",
+      "version": ""
+    },
+    "golangci-lint": {
+      "last_modified": "",
+      "resolved": "github:NixOS/nixpkgs/3364b5b117f65fe1ce65a3cdd5612a078a3b31e3#golangci-lint",
+      "version": ""
+    }
+  }
+}

examples/flakes/remote/devbox.json

@@ -10,4 +10,4 @@
   "nixpkgs": {
     "commit": "f80ac848e3d6f0c12c52758c0f25c10c97ca3b62"
   }
-}
+}

internal/boxcli/featureflag/lockfile.go

@@ -0,0 +1,3 @@
+package featureflag
+
+var LockFile = disabled("LOCKFILE")

internal/boxcli/featureflag/versionedpkgs.go

@@ -140,15 +140,15 @@ func (d *Devbox) ConfigHash() (string, error) {
 	return d.cfg.Hash()
 }
 
+func (d *Devbox) NixPkgsCommitHash() string {
+	return d.cfg.Nixpkgs.Commit
+}
+
 func (d *Devbox) ShellPlan() (*plansdk.ShellPlan, error) {
 	shellPlan := planner.GetShellPlan(d.projectDir, d.packages())
-	shellPlan.DevPackages = lo.Uniq(append(d.localPackages(), shellPlan.DevPackages...))
 	shellPlan.FlakeInputs = d.flakeInputs()
 
-	nixpkgsInfo, err := plansdk.GetNixpkgsInfo(d.cfg.Nixpkgs.Commit)
-	if err != nil {
-		return nil, err
-	}
+	nixpkgsInfo := plansdk.GetNixpkgsInfo(d.cfg.Nixpkgs.Commit)
 	shellPlan.NixpkgsInfo = nixpkgsInfo
 
 	return shellPlan, nil
@@ -628,7 +628,7 @@ func (d *Devbox) StartProcessManager(
 	processComposePath, err := utilityLookPath("process-compose")
 	if err != nil {
 		fmt.Fprintln(d.writer, "Installing process-compose. This may take a minute but will only happen once.")
-		if err = d.addDevboxUtilityPackage("process-compose"); err != nil {
+		if err = d.addDevboxUtilityPackage("github:F1bonacc1/process-compose/v0.43.1"); err != nil {
 			return err
 		}
 
@@ -937,14 +937,6 @@ func (d *Devbox) packages() []string {
 	return d.cfg.Packages
 }
 
-// TODO(landau): localPackages, and flakeInput packages could
-// be merged into a single buildInput map of the form: source => []pkg
-func (d *Devbox) localPackages() []string {
-	return lo.Filter(d.cfg.Packages, func(pkg string, _ int) bool {
-		return !nix.InputFromString(pkg, d.lockfile).IsFlake()
-	})
-}
-
 // configEnvs takes the computed env variables (nix + plugin) and adds env
 // variables defined in Config. It also parses variables in config
 // that are referenced by $VAR or ${VAR} and replaces them with

internal/impl/devbox.go

@@ -10,26 +10,28 @@ import (
 	"go.jetpack.io/devbox/internal/planner/plansdk"
 )
 
+// flakeInputs returns a list of flake inputs for the top level flake.nix
+// created by devbox. We map packages to the correct flake and attribute path
+// and group flakes by URL to avoid duplication. All inputs should be locked
+// i.e. have a commit hash and always resolve to the same package/version.
 func (d *Devbox) flakeInputs() []*plansdk.FlakeInput {
 	inputs := map[string]*plansdk.FlakeInput{}
 	for _, p := range d.packages() {
 		pkg := nix.InputFromString(p, d.lockfile)
-		if pkg.IsFlake() {
-			AttributePath, err := pkg.PackageAttributePath()
-			if err != nil {
-				panic(err)
-			}
-			if input, ok := inputs[pkg.URLForInput()]; !ok {
-				inputs[pkg.URLForInput()] = &plansdk.FlakeInput{
-					Name:     pkg.Name(),
-					URL:      pkg.URLForInput(),
-					Packages: []string{AttributePath},
-				}
-			} else {
-				input.Packages = lo.Uniq(
-					append(inputs[pkg.URLForInput()].Packages, AttributePath),
-				)
+		AttributePath, err := pkg.PackageAttributePath()
+		if err != nil {
+			panic(err)
+		}
+		if input, ok := inputs[pkg.URLForInput()]; !ok {
+			inputs[pkg.URLForInput()] = &plansdk.FlakeInput{
+				Name:     pkg.Name(),
+				URL:      pkg.URLForInput(),
+				Packages: []string{AttributePath},
 			}
+		} else {
+			input.Packages = lo.Uniq(
+				append(inputs[pkg.URLForInput()].Packages, AttributePath),
+			)
 		}
 	}
 

internal/impl/flakes.go

@@ -60,7 +60,7 @@ func (d *Devbox) generateShellFiles() error {
 		}
 	}
 
-	for _, pkg := range plan.DevPackages {
+	for _, pkg := range d.packages() {
 		if err := d.pluginManager.CreateFilesAndShowReadme(d.writer, pkg, d.projectDir); err != nil {
 			return err
 		}

internal/impl/generate.go

@@ -80,7 +80,7 @@ var testFlakeTmplPlan = &struct {
 	NixpkgsInfo struct {
 		URL string
 	}
-	Definitions []string
+	Definitions map[string]string
 	DevPackages []string
 	FlakeInputs []plansdk.FlakeInput
 }{
@@ -89,9 +89,9 @@ var testFlakeTmplPlan = &struct {
 	}{
 		URL: "https://github.com/nixos/nixpkgs/archive/b9c00c1d41ccd6385da243415299b39aa73357be.tar.gz",
 	},
-	Definitions: []string{
-		"php = pkgs.php.withExtensions ({ enabled, all }: enabled ++ (with all; [ blackfire ]));",
-		"php81Packages.composer = php.packages.composer;",
+	Definitions: map[string]string{
+		"php":                    "pkgs.php.withExtensions ({ enabled, all }: enabled ++ (with all; [ blackfire ]));",
+		"php81Packages.composer": "php.packages.composer;",
 	},
 	DevPackages: []string{
 		"php",

internal/impl/generate_test.go

@@ -35,7 +35,7 @@ func (d *Devbox) AddGlobal(pkgs ...string) error {
 
 	// validate all packages exist. Don't install anything if any are missing
 	for _, pkg := range pkgs {
-		found, err := nix.PkgExists(plansdk.DefaultNixpkgsCommit, pkg, d.lockfile)
+		found, err := nix.PkgExists(pkg, d.lockfile)
 		if err != nil {
 			return err
 		} else if !found {
@@ -55,7 +55,6 @@ func (d *Devbox) AddGlobal(pkgs ...string) error {
 		err = nix.ProfileInstall(&nix.ProfileInstallArgs{
 			CustomStepMessage: stepMsg,
 			Lockfile:          d.lockfile,
-			NixpkgsCommit:     d.cfg.Nixpkgs.Commit,
 			Package:           pkg,
 			ProfilePath:       profilePath,
 			Writer:            d.writer,

internal/impl/global.go

@@ -39,7 +39,7 @@ func (d *Devbox) Add(ctx context.Context, pkgs ...string) error {
 
 	// Check packages are valid before adding.
 	for _, pkg := range pkgs {
-		ok, err := nix.PkgExists(d.cfg.Nixpkgs.Commit, pkg, d.lockfile)
+		ok, err := nix.PkgExists(pkg, d.lockfile)
 		if err != nil {
 			return err
 		}
@@ -237,7 +237,6 @@ func (d *Devbox) addPackagesToProfile(ctx context.Context, mode installMode) err
 		if err := nix.ProfileInstall(&nix.ProfileInstallArgs{
 			CustomStepMessage: stepMsg,
 			Lockfile:          d.lockfile,
-			NixpkgsCommit:     d.cfg.Nixpkgs.Commit,
 			Package:           pkg,
 			ProfilePath:       profileDir,
 			Writer:            d.writer,