[planner->plugin] Turn php planner into a plugin (#1000)

## Summary

This PR turns our PHP planner into a plugin. Our plugins are actually
almost able to do this with some minimal changes.

Re-used existing functionality:

* Create `flake.nix` file in the php plugin
* packages (inputs) can be local flakes

Changes:

* Pass `Packages`, `System`, and `URLForInput` to plugin file templates.
This allows plugins to build and customize some interesting flakes.
* Add `packages` field to plugins. We wanted to do this for a while. In
this case it simply points to the flake created by the plugin. These
packages are added to the devbox environment.

Semi-related changes:

* Use `--recreate-lock-file` every time we `print-dev-env`. The reason
this is needed is because input flakes might change se we need to
frequently update. It doesn't hurt performance much because we cache
print-dev-env. This means any unlocked package will update frequently. I
think that's fine because this lockfile is not shared so there's no
expectation of stability.

cc: @Lagoja @savil 

## How was it tested?

```bash
devbox add php phpExtensions.ds
devbox run 'php -m | grep ds'
```

Author: mikeland73

internal/cuecfg/hash.go

@@ -13,7 +13,10 @@ import (
 
 func FileHash(path string) (string, error) {
 	data, err := os.ReadFile(path)
-	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+	if errors.Is(err, fs.ErrNotExist) {
+		return "", nil
+	}
+	if err != nil {
 		return "", err
 	}
 	hash := sha256.Sum256(data)

internal/impl/devbox.go

@@ -125,7 +125,10 @@ func Open(path string, writer io.Writer) (*Devbox, error) {
 	if err != nil {
 		return nil, err
 	}
-	box.pluginManager.ApplyOptions(plugin.WithLockfile(lock))
+	box.pluginManager.ApplyOptions(
+		plugin.WithDevbox(box),
+		plugin.WithLockfile(lock),
+	)
 	box.lockfile = lock
 	return box, nil
 }
@@ -152,7 +155,26 @@ func (d *Devbox) NixPkgsCommitHash() string {
 }
 
 func (d *Devbox) ShellPlan() (*plansdk.ShellPlan, error) {
-	shellPlan := planner.GetShellPlan(d.projectDir, d.packages())
+	// Create plugin directories first because inputs might depend on them
+	for _, pkg := range d.packagesAsInputs() {
+		if err := d.pluginManager.Create(d.writer, pkg); err != nil {
+			return nil, err
+		}
+	}
+
+	for _, included := range d.cfg.Include {
+		// This is a slightly weird place to put this, but since includes can't be
+		// added via command and we need them to be added before we call
+		// plugin manager.Include
+		if err := d.lockfile.Add(included); err != nil {
+			return nil, err
+		}
+		if err := d.pluginManager.Include(d.writer, included); err != nil {
+			return nil, err
+		}
+	}
+
+	shellPlan := planner.GetShellPlan(d.projectDir, d.Packages())
 	var err error
 	shellPlan.FlakeInputs, err = d.flakeInputs()
 	if err != nil {
@@ -369,7 +391,7 @@ func (d *Devbox) GenerateDevcontainer(force bool) error {
 			redact.Safe(filepath.Base(devContainerPath)), err)
 	}
 	// generate devcontainer.json
-	err = generate.CreateDevcontainer(devContainerPath, d.packages())
+	err = generate.CreateDevcontainer(devContainerPath, d.Packages())
 	if err != nil {
 		return redact.Errorf("error generating devcontainer.json in <project>/%s: %w",
 			redact.Safe(filepath.Base(devContainerPath)), err)
@@ -452,7 +474,6 @@ func (d *Devbox) Services() (services.Services, error) {
 	pluginSvcs, err := d.pluginManager.GetServices(
 		d.packagesAsInputs(),
 		d.cfg.Include,
-		d.projectDir,
 	)
 	if err != nil {
 		return nil, err
@@ -787,8 +808,7 @@ func (d *Devbox) computeNixEnv(ctx context.Context, usePrintDevEnvCache bool) (m
 	// We still need to be able to add env variables to non-service binaries
 	// (e.g. ruby). This would involve understanding what binaries are associated
 	// to a given plugin.
-	pluginEnv, err := d.pluginManager.Env(
-		d.packagesAsInputs(), d.cfg.Include, d.projectDir, env)
+	pluginEnv, err := d.pluginManager.Env(d.packagesAsInputs(), d.cfg.Include, env)
 	if err != nil {
 		return nil, err
 	}
@@ -955,13 +975,13 @@ func (d *Devbox) nixFlakesFilePath() string {
 	return filepath.Join(d.projectDir, ".devbox/gen/flake/flake.nix")
 }
 
-// packages returns the list of packages to be installed in the nix shell.
-func (d *Devbox) packages() []string {
+// Packages returns the list of Packages to be installed in the nix shell.
+func (d *Devbox) Packages() []string {
 	return d.cfg.Packages
 }
 
 func (d *Devbox) packagesAsInputs() []*nix.Input {
-	return nix.InputsFromStrings(d.packages(), d.lockfile)
+	return nix.InputsFromStrings(d.Packages(), d.lockfile)
 }
 
 func (d *Devbox) findPackageByName(name string) (string, error) {

internal/impl/devbox_test.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/bmatcuk/doublestar/v4"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"go.jetpack.io/devbox/internal/envir"
 	"go.jetpack.io/devbox/internal/fileutil"
@@ -25,7 +26,7 @@ func TestDevbox(t *testing.T) {
 	t.Setenv("TMPDIR", "/tmp")
 	t.Setenv(envir.DevboxDoNotUpgradeConfig, "1")
 	testPaths, err := doublestar.FilepathGlob("../../examples/**/devbox.json")
-	assert.NoError(t, err, "Reading testdata/ should not fail")
+	require.NoError(t, err, "Reading testdata/ should not fail")
 
 	assert.Greater(t, len(testPaths), 0, "testdata/ and examples/ should contain at least 1 test")
 
@@ -78,25 +79,28 @@ func (n *testNix) PrintDevEnv(ctx context.Context, args *nix.PrintDevEnvArgs) (*
 }
 
 func TestComputeNixEnv(t *testing.T) {
-	d := &Devbox{
-		cfg: &Config{},
-		nix: &testNix{},
-	}
+	path := t.TempDir()
+	_, err := InitConfig(path, os.Stdout)
+	require.NoError(t, err, "InitConfig should not fail")
+	d, err := Open(path, os.Stdout)
+	require.NoError(t, err, "Open should not fail")
+	d.nix = &testNix{}
 	ctx := context.Background()
 	env, err := d.computeNixEnv(ctx, false /*use cache*/)
-	assert.NoError(t, err, "computeNixEnv should not fail")
+	require.NoError(t, err, "computeNixEnv should not fail")
 	assert.NotNil(t, env, "computeNixEnv should return a valid env")
 }
 
 func TestComputeNixPathIsIdempotent(t *testing.T) {
-	devbox := &Devbox{
-		cfg:        &Config{},
-		nix:        &testNix{"/tmp/my/path"},
-		projectDir: "/tmp/TestComputeNixPathIsIdempotent",
-	}
+	dir := t.TempDir()
+	_, err := InitConfig(dir, os.Stdout)
+	require.NoError(t, err, "InitConfig should not fail")
+	devbox, err := Open(dir, os.Stdout)
+	require.NoError(t, err, "Open should not fail")
+	devbox.nix = &testNix{"/tmp/my/path"}
 	ctx := context.Background()
 	env, err := devbox.computeNixEnv(ctx, false /*use cache*/)
-	assert.NoError(t, err, "computeNixEnv should not fail")
+	require.NoError(t, err, "computeNixEnv should not fail")
 	path := env["PATH"]
 	assert.NotEmpty(t, path, "path should not be nil")
 
@@ -107,21 +111,22 @@ func TestComputeNixPathIsIdempotent(t *testing.T) {
 	)
 
 	env, err = devbox.computeNixEnv(ctx, false /*use cache*/)
-	assert.NoError(t, err, "computeNixEnv should not fail")
+	require.NoError(t, err, "computeNixEnv should not fail")
 	path2 := env["PATH"]
 
 	assert.Equal(t, path, path2, "path should be the same")
 }
 
 func TestComputeNixPathWhenRemoving(t *testing.T) {
-	devbox := &Devbox{
-		cfg:        &Config{},
-		nix:        &testNix{"/tmp/my/path"},
-		projectDir: "/tmp/TestComputeNixPathWhenRemoving",
-	}
+	dir := t.TempDir()
+	_, err := InitConfig(dir, os.Stdout)
+	require.NoError(t, err, "InitConfig should not fail")
+	devbox, err := Open(dir, os.Stdout)
+	require.NoError(t, err, "Open should not fail")
+	devbox.nix = &testNix{"/tmp/my/path"}
 	ctx := context.Background()
 	env, err := devbox.computeNixEnv(ctx, false /*use cache*/)
-	assert.NoError(t, err, "computeNixEnv should not fail")
+	require.NoError(t, err, "computeNixEnv should not fail")
 	path := env["PATH"]
 	assert.NotEmpty(t, path, "path should not be nil")
 	assert.Contains(t, path, "/tmp/my/path", "path should contain /tmp/my/path")
@@ -134,7 +139,7 @@ func TestComputeNixPathWhenRemoving(t *testing.T) {
 
 	devbox.nix.(*testNix).path = ""
 	env, err = devbox.computeNixEnv(ctx, false /*use cache*/)
-	assert.NoError(t, err, "computeNixEnv should not fail")
+	require.NoError(t, err, "computeNixEnv should not fail")
 	path2 := env["PATH"]
 	assert.NotContains(t, path2, "/tmp/my/path", "path should not contain /tmp/my/path")
 

internal/impl/flakes.go

@@ -6,25 +6,37 @@ package impl
 import (
 	"github.com/samber/lo"
 
-	"go.jetpack.io/devbox/internal/nix"
 	"go.jetpack.io/devbox/internal/planner/plansdk"
 )
 
 // flakeInputs returns a list of flake inputs for the top level flake.nix
 // created by devbox. We map packages to the correct flake and attribute path
 // and group flakes by URL to avoid duplication. All inputs should be locked
 // i.e. have a commit hash and always resolve to the same package/version.
+// Note: inputs returned by this function include plugin packages. (php only for now)
+// It's not entirely clear we always want to add plugin packages to the top level
 func (d *Devbox) flakeInputs() ([]*plansdk.FlakeInput, error) {
 	inputs := map[string]*plansdk.FlakeInput{}
-	for _, p := range d.packages() {
-		pkg := nix.InputFromString(p, d.lockfile)
+
+	userPackages := d.packagesAsInputs()
+	pluginPackages, err := d.pluginManager.PluginPackages(userPackages)
+	if err != nil {
+		return nil, err
+	}
+
+	order := []string{}
+	// We prioritize plugin packages so that the php plugin works. Not sure
+	// if this is behavior we want for user plugins. We may need to add an optional
+	// priority field to the config.
+	for _, pkg := range append(pluginPackages, userPackages...) {
 		AttributePath, err := pkg.PackageAttributePath()
 		if err != nil {
 			return nil, err
 		}
 		if input, ok := inputs[pkg.URLForInput()]; !ok {
+			order = append(order, pkg.URLForInput())
 			inputs[pkg.URLForInput()] = &plansdk.FlakeInput{
-				Name:     pkg.Name(),
+				Name:     pkg.InputName(),
 				URL:      pkg.URLForInput(),
 				Packages: []string{AttributePath},
 			}
@@ -35,5 +47,14 @@ func (d *Devbox) flakeInputs() ([]*plansdk.FlakeInput, error) {
 		}
 	}
 
-	return lo.Values(inputs), nil
+	return PickByKeysSorted(inputs, order), nil
+}
+
+// TODO: move this to a util package
+func PickByKeysSorted[K comparable, V any](in map[K]V, keys []K) []V {
+	out := make([]V, len(keys))
+	for i, key := range keys {
+		out[i] = in[key]
+	}
+	return out
 }

internal/impl/generate.go

@@ -60,21 +60,6 @@ func (d *Devbox) generateShellFiles() error {
 		}
 	}
 
-	for _, pkg := range d.packagesAsInputs() {
-		if err := d.pluginManager.Create(d.writer, pkg, d.projectDir); err != nil {
-			return err
-		}
-	}
-
-	for _, included := range d.cfg.Include {
-		if err := d.lockfile.Add(included); err != nil {
-			return err
-		}
-		if err := d.pluginManager.Include(d.writer, included, d.projectDir); err != nil {
-			return err
-		}
-	}
-
 	return d.writeScriptsToFiles()
 }
 

internal/impl/packages.go

@@ -314,7 +314,7 @@ func (d *Devbox) pendingPackagesForInstallation(ctx context.Context) ([]string,
 	if err != nil {
 		return nil, err
 	}
-	for _, pkg := range d.packages() {
+	for _, pkg := range d.Packages() {
 		_, err := nix.ProfileListIndex(&nix.ProfileListIndexArgs{
 			List:       list,
 			Lockfile:   d.lockfile,

internal/impl/update.go

@@ -17,7 +17,7 @@ func (d *Devbox) Update(ctx context.Context, pkgs ...string) error {
 		pkgsToUpdate = append(pkgsToUpdate, found)
 	}
 	if len(pkgsToUpdate) == 0 {
-		pkgsToUpdate = d.packages()
+		pkgsToUpdate = d.Packages()
 	}
 
 	for _, pkg := range pkgsToUpdate {

internal/nix/input.go

@@ -16,6 +16,7 @@ import (
 
 	"go.jetpack.io/devbox/internal/boxcli/featureflag"
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
+	"go.jetpack.io/devbox/internal/cuecfg"
 	"go.jetpack.io/devbox/internal/lock"
 	"go.jetpack.io/devbox/internal/searcher"
 )
@@ -60,7 +61,7 @@ func (i *Input) IsGithub() bool {
 
 var inputNameRegex = regexp.MustCompile("[^a-zA-Z0-9-]+")
 
-func (i *Input) Name() string {
+func (i *Input) InputName() string {
 	result := ""
 	if i.IsLocal() {
 		result = filepath.Base(i.Path) + "-" + i.hash()
@@ -175,6 +176,14 @@ func (i *Input) urlWithoutFragment() string {
 }
 
 func (i *Input) hash() string {
+	// For local flakes, use content hash of the flake.nix file to ensure
+	// user always gets newest input.
+	if i.IsLocal() {
+		fileHash, _ := cuecfg.FileHash(filepath.Join(i.Path, "flake.nix"))
+		if fileHash != "" {
+			return fileHash[:6]
+		}
+	}
 	hasher := md5.New()
 	hasher.Write([]byte(i.String()))
 	hash := hasher.Sum(nil)

internal/nix/input_test.go

@@ -86,7 +86,7 @@ func TestInput(t *testing.T) {
 
 	for _, testCase := range cases {
 		i := testInputFromString(testCase.pkg, projectDir)
-		if name := i.Name(); testCase.name != name {
+		if name := i.InputName(); testCase.name != name {
 			t.Errorf("Name() = %v, want %v", name, testCase.name)
 		}
 		if urlWithoutFragment := i.urlWithoutFragment(); testCase.urlWithoutFragment != urlWithoutFragment {

internal/nix/nix.go

@@ -58,7 +58,11 @@ func (*Nix) PrintDevEnv(ctx context.Context, args *PrintDevEnvArgs) (*PrintDevEn
 	}
 
 	if len(data) == 0 {
-		cmd := exec.CommandContext(ctx, "nix", "print-dev-env", args.FlakesFilePath)
+		cmd := exec.CommandContext(
+			ctx,
+			"nix", "print-dev-env",
+			args.FlakesFilePath,
+		)
 		cmd.Args = append(cmd.Args, ExperimentalFlags()...)
 		cmd.Args = append(cmd.Args, "--json")
 		debug.Log("Running print-dev-env cmd: %s\n", cmd)
@@ -104,6 +108,23 @@ func ExperimentalFlags() []string {
 	}
 }
 
+var cachedSystem string
+
+func System() (string, error) {
+	if cachedSystem == "" {
+		cmd := exec.Command(
+			"nix", "eval", "--impure", "--raw", "--expr", "builtins.currentSystem",
+		)
+		cmd.Args = append(cmd.Args, ExperimentalFlags()...)
+		out, err := cmd.Output()
+		if err != nil {
+			return "", errors.WithStack(err)
+		}
+		cachedSystem = string(out)
+	}
+	return cachedSystem, nil
+}
+
 // Warning: be careful using the bins in default/bin, they won't always match bins
 // produced by the flakes.nix. Use devbox.NixBins() instead.
 func ProfileBinPath(projectDir string) string {