[lockfile] Add hidden lockfile for caching (#897)

## Summary

This adds a hidden lock file for caching purposes. 

### Motivation:
This speeds up run/shell by only running `ensurePackagesAreInstalled` if config or nix profile manifest has changed.

### Why hidden?
I'm not ready for this to be an official checked-in lock file. So for now it lives in `.devbox` and does not get committed to repo.

## How was it tested?

In flakes example ran:

```bash
devbox run echo 5
```

and got a speed up of around 66%.

Also tested

```bash
cat .devbox/devbox.lock
devbox add hello
cat .devbox/devbox.lock  # does not match original
devbox run hello #success
devbox rm hello
cat .devbox/devbox.lock  # matches original
devbox run hello # error
```

Author: mikeland73

internal/cuecfg/cuecfg.go

@@ -20,7 +20,7 @@ func Marshal(valuePtr any, extension string) ([]byte, error) {
 	}
 
 	switch extension {
-	case ".json":
+	case ".json", ".lock":
 		return MarshalJSON(valuePtr)
 	case ".yml", ".yaml":
 		return marshalYaml(valuePtr)
@@ -34,7 +34,7 @@ func Marshal(valuePtr any, extension string) ([]byte, error) {
 
 func Unmarshal(data []byte, extension string, valuePtr any) error {
 	switch extension {
-	case ".json":
+	case ".json", ".lock":
 		return errors.WithStack(unmarshalJSON(data, valuePtr))
 	case ".yml", ".yaml":
 		return errors.WithStack(unmarshalYaml(data, valuePtr))
@@ -87,7 +87,7 @@ func WriteFile(path string, value any) error {
 
 func IsSupportedExtension(ext string) bool {
 	switch ext {
-	case ".json", ".yml", ".yaml", ".toml", ".xml":
+	case ".json", ".lock", ".yml", ".yaml", ".toml", ".xml":
 		return true
 	default:
 		return false

internal/impl/config.go

@@ -4,6 +4,8 @@
 package impl
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"io"
 	"net/http"
 	"net/url"
@@ -66,6 +68,15 @@ func (c *Config) MergedPackages(w io.Writer) []string {
 	return lo.Uniq(append(c.RawPackages, global.RawPackages...))
 }
 
+func (c *Config) Hash() (string, error) {
+	json, err := cuecfg.MarshalJSON(c)
+	if err != nil {
+		return "", err
+	}
+	hash := sha256.Sum256(json)
+	return hex.EncodeToString(hash[:]), nil
+}
+
 func readConfig(path string) (*Config, error) {
 	cfg := &Config{}
 	return cfg, errors.WithStack(cuecfg.ParseFile(path, cfg))

internal/impl/devbox.go

@@ -130,6 +130,10 @@ func (d *Devbox) Config() *Config {
 	return d.cfg
 }
 
+func (d *Devbox) ConfigHash() (string, error) {
+	return d.cfg.Hash()
+}
+
 func (d *Devbox) ShellPlan() (*plansdk.ShellPlan, error) {
 	shellPlan := planner.GetShellPlan(d.projectDir, d.mergedPackages())
 	shellPlan.DevPackages = pkgslice.Unique(append(d.localPackages(), shellPlan.DevPackages...))

internal/impl/packages.go

@@ -13,6 +13,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/samber/lo"
 	"go.jetpack.io/devbox/internal/debug"
+	"go.jetpack.io/devbox/internal/lockfile"
 	"go.jetpack.io/devbox/internal/nix"
 	"go.jetpack.io/devbox/internal/plugin"
 	"go.jetpack.io/devbox/internal/ux"
@@ -136,6 +137,17 @@ const (
 func (d *Devbox) ensurePackagesAreInstalled(ctx context.Context, mode installMode) error {
 	defer trace.StartRegion(ctx, "ensurePackages").End()
 
+	lock, err := lockfile.Local(d)
+	if err != nil {
+		return err
+	}
+
+	if upToDate, err := lock.IsUpToDate(); err != nil {
+		return err
+	} else if upToDate {
+		return nil
+	}
+
 	if err := d.generateShellFiles(); err != nil {
 		return err
 	}
@@ -147,7 +159,11 @@ func (d *Devbox) ensurePackagesAreInstalled(ctx context.Context, mode installMod
 		return err
 	}
 
-	return plugin.RemoveInvalidSymlinks(d.projectDir)
+	if err := plugin.RemoveInvalidSymlinks(d.projectDir); err != nil {
+		return err
+	}
+
+	return lock.Update()
 }
 
 func (d *Devbox) profilePath() (string, error) {

internal/lockfile/lockfile.go

@@ -0,0 +1,85 @@
+package lockfile
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+
+	"go.jetpack.io/devbox/internal/cuecfg"
+	"go.jetpack.io/devbox/internal/nix"
+)
+
+// localLockFile is a non-shared lock file that helps track the state of the
+// local devbox environment. It contains hashes that may not be the same across
+// machines (e.g. manifest hash).
+// When we do implement a shared lock file, it may contain some shared fields
+// with this one but not all.
+type localLockFile struct {
+	project                devboxProject
+	ConfigHash             string `json:"config_hash"`
+	NixProfileManifestHash string `json:"nix_profile_manifest_hash"`
+}
+
+func (l *localLockFile) equals(other *localLockFile) bool {
+	return l.ConfigHash == other.ConfigHash &&
+		l.NixProfileManifestHash == other.NixProfileManifestHash
+}
+
+func (l *localLockFile) IsUpToDate() (bool, error) {
+	newLock, err := forProject(l.project)
+	if err != nil {
+		return false, err
+	}
+
+	return l.equals(newLock), nil
+}
+
+func (l *localLockFile) Update() error {
+	newLock, err := forProject(l.project)
+	if err != nil {
+		return err
+	}
+	*l = *newLock
+
+	return cuecfg.WriteFile(localLockFilePath(l.project), l)
+}
+
+type devboxProject interface {
+	ConfigHash() (string, error)
+	ProjectDir() string
+}
+
+func Local(project devboxProject) (*localLockFile, error) {
+	lockFile := &localLockFile{project: project}
+	err := cuecfg.ParseFile(localLockFilePath(project), lockFile)
+	if errors.Is(err, os.ErrNotExist) {
+		return lockFile, nil
+	} else if err != nil {
+		return nil, err
+	}
+	return lockFile, nil
+}
+
+func forProject(project devboxProject) (*localLockFile, error) {
+	configHash, err := project.ConfigHash()
+	if err != nil {
+		return nil, err
+	}
+
+	nixHash, err := nix.ManifestHash(project.ProjectDir())
+	if err != nil {
+		return nil, err
+	}
+
+	newLock := &localLockFile{
+		project:                project,
+		ConfigHash:             configHash,
+		NixProfileManifestHash: nixHash,
+	}
+
+	return newLock, nil
+}
+
+func localLockFilePath(project devboxProject) string {
+	return filepath.Join(project.ProjectDir(), ".devbox", "local.lock")
+}

internal/nix/profile.go

@@ -3,6 +3,8 @@ package nix
 import (
 	"bufio"
 	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -331,6 +333,16 @@ func readManifest(profilePath string) (manifest, error) {
 	return m, json.Unmarshal(data, &m)
 }
 
+func ManifestHash(profileDir string) (string, error) {
+	path := filepath.Join(profileDir, ProfilePath, "manifest.json")
+	data, err := os.ReadFile(path)
+	if err != nil && !os.IsNotExist(err) {
+		return "", err
+	}
+	hash := sha256.Sum256(data)
+	return hex.EncodeToString(hash[:]), nil
+}
+
 func nextPriority(profilePath string) string {
 	// error is ignored because it's ok if the file doesn't exist
 	m, _ := readManifest(profilePath)