[remove nixpkgs] add toPath, and edit devbox.lock only on update command (#1256)

## Summary

This PR rolls in a few fixes:
1. Add `toPath` to `builtins.fetchClosure`
    - lets us remove `--impure` flag in `nix print-dev-env`
2. Add `ca_store_path` to `devbox.lock`
- Because this is system-dependent, _and_ is calculated locally, a
`devbox update` will only update the user's system's `ca_store_path`.
- This is good for team-mates on the same system, and to avoid the slow
`nix store make-content-addressed <path>` call on each `devbox
shell/run/shellenv`.
3. Only update `devbox.lock` during `devbox update`. 
- Previously, it would update system info on `devbox
shell/run/shellenv`.
4. Simplify `devbox.lock` schema's `SystemInfo` to have `StorePath` and
`CAStorePath`. These are used as `fromPath` and `toPath` in
`builtins.fetchClosure`.

Miscellaneous changes:
1. I remove `actionlint` from the `devbox.json`. It is not core to our
workflows, and I am working with very minimal space on devbox cloud VM.
This reduces "out of space errors" marginally.

## How was it tested?

In `examples/development/go/hello-world`:
`devbox update` and `devbox shell`


The `devbox.lock` diff is now 
```
❯ git diff
diff --git a/examples/development/go/hello-world/devbox.lock b/examples/development/go/hello-world/
devbox.lock
index 34556d3..168874db 100644
--- a/examples/development/go/hello-world/devbox.lock
+++ b/examples/development/go/hello-world/devbox.lock
@@ -2,9 +2,18 @@
   "lockfile_version": "1",
   "packages": {
     "[email protected]": {
-      "last_modified": "2023-05-01T16:53:22Z",
-      "resolved": "github:NixOS/nixpkgs/8670e496ffd093b60e74e7fa53526aa5920d09eb#go_1_19",
-      "version": "1.19.8"
+      "last_modified": "2023-05-03T02:55:54Z",
+      "resolved": "github:NixOS/nixpkgs/0d373d5af960504dd60c3d06c65e553b36ef29d8#go_1_19",

+      "version": "1.19.8",
+      "systems": {
+        "aarch64-darwin": {
+          "store_path": "/nix/store/7m99ip3616l3z670y83p34r3plwd4iq1-go-1.19.8"
+        },
+        "x86_64-linux": {
+          "store_path": "/nix/store/r0x0agq0vwn0p6z99vkkvn8l8a8idzsb-go-1.19.8",
+          "ca_store_path": "/nix/store/ns557l24yf1f16f9jvbmxv57qdkkn2mj-go-1.19.8"
+        }
+      }
     }
   }
-}
\ No newline at end of file
+}
```

Author: savil

devbox.json

@@ -1,7 +1,6 @@
 {
   "packages": [
     "[email protected]",
-    "[email protected]",
     "[email protected]"
   ],
   "env": {

devbox.lock

@@ -1,11 +1,6 @@
 {
   "lockfile_version": "1",
   "packages": {
-    "[email protected]": {
-      "last_modified": "2023-03-31T22:52:29Z",
-      "resolved": "github:NixOS/nixpkgs/242246ee1e58f54d2322227fc5eef53b4a616a31#actionlint",
-      "version": "1.6.23"
-    },
     "[email protected]": {
       "last_modified": "2023-05-25T03:54:59Z",
       "resolved": "github:NixOS/nixpkgs/8d4d822bc0efa9de6eddc79cb0d82897a9baa750#go",

internal/devpkg/package.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io"
 	"net/url"
+	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -20,6 +21,7 @@ import (
 	"go.jetpack.io/devbox/internal/cuecfg"
 	"go.jetpack.io/devbox/internal/lock"
 	"go.jetpack.io/devbox/internal/nix"
+	"go.jetpack.io/devbox/internal/ux"
 )
 
 // Package represents a "package" added to the devbox.json config.
@@ -426,8 +428,10 @@ func (p *Package) HashFromNixPkgsURL() string {
 
 // BinaryCacheStore is the store from which to fetch this package's binaries.
 // It is used as FromStore in builtins.fetchClosure.
+// TODO savil: rename to BinaryCache
 const BinaryCacheStore = "https://cache.nixos.org"
 
+// TODO savil: rename to IsInBinaryCache
 func (p *Package) IsInBinaryStore() (bool, error) {
 	if !featureflag.RemoveNixpkgs.Enabled() {
 		return false, nil
@@ -457,7 +461,8 @@ func (p *Package) IsInBinaryStore() (bool, error) {
 }
 
 // PathInBinaryStore is the key in the BinaryCacheStore for this package
-// This is used as FromPath in builtins.fetchClosure
+// This is used as StorePath in builtins.fetchClosure
+// TODO savil: rename to PathInBinaryCache
 func (p *Package) PathInBinaryStore() (string, error) {
 	if isInStore, err := p.IsInBinaryStore(); err != nil {
 		return "", err
@@ -477,10 +482,41 @@ func (p *Package) PathInBinaryStore() (string, error) {
 	}
 
 	sysInfo := entry.Systems[userSystem]
-	storeDirParts := []string{sysInfo.FromHash, sysInfo.StoreName}
-	if sysInfo.StoreVersion != "" {
-		storeDirParts = append(storeDirParts, sysInfo.StoreVersion)
+	return sysInfo.StorePath, nil
+}
+
+func (p *Package) ContentAddressedStorePath() (string, error) {
+
+	if isInStore, err := p.IsInBinaryStore(); err != nil {
+		return "", err
+	} else if !isInStore {
+		return "",
+			errors.Errorf("Package %q cannot be fetched from binary cache store", p.Raw)
+	}
+
+	entry, err := p.lockfile.Resolve(p.Raw)
+	if err != nil {
+		return "", err
+	}
+
+	userSystem, err := nix.System()
+	if err != nil {
+		return "", err
+	}
+
+	sysInfo := entry.Systems[userSystem]
+	if sysInfo.CAStorePath != "" {
+		return sysInfo.CAStorePath, nil
+	}
+
+	ux.Fwarning(
+		os.Stderr,
+		"calculating local_store_path. This may be slow.\n"+
+			"Run `devbox update` to speed this up for next time.",
+	)
+	localPath, err := nix.ContentAddressedStorePath(sysInfo.StorePath)
+	if err != nil {
+		return "", err
 	}
-	storeDir := strings.Join(storeDirParts, "-")
-	return filepath.Join("/nix/store", storeDir), nil
+	return localPath, err
 }

internal/impl/update.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 
+	"go.jetpack.io/devbox/internal/boxcli/featureflag"
 	"go.jetpack.io/devbox/internal/devpkg"
 	"go.jetpack.io/devbox/internal/nix"
 	"go.jetpack.io/devbox/internal/nix/nixprofile"
@@ -90,21 +91,50 @@ func (d *Devbox) updateDevboxPackage(
 	if err != nil {
 		return err
 	}
-	if existing != nil && existing.Version != newEntry.Version {
+	if existing == nil {
+		ux.Finfo(d.writer, "Resolved %s to %[1]s %[2]s\n", pkg, newEntry.Resolved)
+		d.lockfile.Packages[pkg.Raw] = newEntry
+		return nil
+	}
+
+	if existing.Version != newEntry.Version {
 		ux.Finfo(d.writer, "Updating %s %s -> %s\n", pkg, existing.Version, newEntry.Version)
 		if err := d.removePackagesFromProfile(ctx, []string{pkg.Raw}); err != nil {
 			// Warn but continue. TODO(landau): ensurePackagesAreInstalled should
 			// sync the profile so we don't need to do this manually.
 			ux.Fwarning(d.writer, "Failed to remove %s from profile: %s\n", pkg, err)
 		}
 		d.lockfile.Packages[pkg.Raw] = newEntry
-	} else if existing == nil {
-		ux.Finfo(d.writer, "Resolved %s to %[1]s %[2]s\n", pkg, newEntry.Resolved)
-		d.lockfile.Packages[pkg.Raw] = newEntry
-	} else {
-		ux.Finfo(d.writer, "Already up-to-date %s %s\n", pkg, existing.Version)
+		return nil
+	}
+
+	// Check if the package's system info is missing, or not complete.
+	if featureflag.RemoveNixpkgs.Enabled() {
+		userSystem, err := nix.System()
+		if err != nil {
+			return err
+		}
+
+		// Check if the system info is missing for the user's system.
+		sysInfo := d.lockfile.Packages[pkg.Raw].Systems[userSystem]
+		if sysInfo == nil {
+			d.lockfile.Packages[pkg.Raw] = newEntry
+			ux.Finfo(d.writer, "Updated system information for %s\n", pkg)
+			return nil
+		}
+
+		// Check if the CAStorePath is missing for the user's system.
+		// Since any one user cannot add this field for all systems,
+		// we'll need to progressively add it to a project's lockfile.
+		if sysInfo.CAStorePath == "" {
+			// Update the CAStorePath for the user's system
+			d.lockfile.Packages[pkg.Raw].Systems[userSystem].CAStorePath = newEntry.Systems[userSystem].CAStorePath
+			ux.Finfo(d.writer, "Updated system information for %s\n", pkg)
+			return nil
+		}
 	}
 
+	ux.Finfo(d.writer, "Already up-to-date %s %s\n", pkg, existing.Version)
 	return nil
 }
 

internal/lock/lockfile.go

@@ -11,8 +11,6 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/samber/lo"
-	"go.jetpack.io/devbox/internal/boxcli/featureflag"
-	"go.jetpack.io/devbox/internal/nix"
 	"go.jetpack.io/devbox/internal/searcher"
 
 	"go.jetpack.io/devbox/internal/cuecfg"
@@ -40,12 +38,13 @@ type Package struct {
 }
 
 type SystemInfo struct {
-	System   string // stored elsewhere in json: it's the key for the Package.Systems
-	FromHash string `json:"from_hash,omitempty"`
-	// StoreName may be different from the canonicalName so we store it separately
-	StoreName    string `json:"store_name,omitempty"`
-	StoreVersion string `json:"store_version,omitempty"`
-	ToHash       string `json:"to_hash,omitempty"`
+	// StorePath is the cache key in the Binary Cache Store (cache.nixos.org)
+	// It is of the form <hash>-<name>-<version>
+	// <name> may be different from the canonicalName so we store the full store path.
+	StorePath string `json:"store_path,omitempty"`
+	// CAStorePath is the content-addressed path for the nix package in /nix/store
+	// It is of the form <hash>-<name>-<version>
+	CAStorePath string `json:"ca_store_path,omitempty"`
 }
 
 func GetFile(project devboxProject) (*File, error) {
@@ -86,17 +85,7 @@ func (l *File) Remove(pkgs ...string) error {
 func (l *File) Resolve(pkg string) (*Package, error) {
 	entry, hasEntry := l.Packages[pkg]
 
-	// If the package's system info is missing, we need to resolve it again.
-	needsSysInfo := false
-	if hasEntry && featureflag.RemoveNixpkgs.Enabled() {
-		userSystem, err := nix.System()
-		if err != nil {
-			return nil, err
-		}
-		needsSysInfo = entry.Systems[userSystem] == nil
-	}
-
-	if !hasEntry || entry.Resolved == "" || needsSysInfo {
+	if !hasEntry || entry.Resolved == "" {
 		locked := &Package{}
 		var err error
 		if _, _, versioned := searcher.ParseVersionedPackage(pkg); versioned {
@@ -109,19 +98,6 @@ func (l *File) Resolve(pkg string) (*Package, error) {
 			// whatever hash is in the devbox.json
 			locked = &Package{Resolved: l.LegacyNixpkgsPath(pkg)}
 		}
-
-		// Merge the system info from the lockfile with the queried system info.
-		// This is necessary because a different system's info may previously have
-		// been added. For example: `aarch64-darwin` was already added, but
-		// current user is on `x86_64-linux`.
-		if hasEntry && featureflag.RemoveNixpkgs.Enabled() {
-			for _, sysInfo := range entry.Systems {
-				if _, ok := locked.Systems[sysInfo.System]; !ok {
-					locked.Systems[sysInfo.System] = sysInfo
-				}
-			}
-		}
-
 		l.Packages[pkg] = locked
 	}
 

internal/lock/resolve.go

@@ -34,7 +34,10 @@ func (l *File) FetchResolvedPackage(pkg string) (*Package, error) {
 
 	sysInfos := map[string]*SystemInfo{}
 	if featureflag.RemoveNixpkgs.Enabled() {
-		sysInfos = buildLockSystemInfos(packageVersion)
+		sysInfos, err = buildLockSystemInfos(packageVersion)
+		if err != nil {
+			return nil, err
+		}
 	}
 	packageInfo, err := selectForSystem(packageVersion)
 	if err != nil {
@@ -76,15 +79,32 @@ func selectForSystem(pkg *searcher.PackageVersion) (searcher.PackageInfo, error)
 	return maps.Values(pkg.Systems)[0], nil
 }
 
-func buildLockSystemInfos(pkg *searcher.PackageVersion) map[string]*SystemInfo {
+func buildLockSystemInfos(pkg *searcher.PackageVersion) (map[string]*SystemInfo, error) {
+	userSystem, err := nix.System()
+	if err != nil {
+		return nil, err
+	}
+
 	sysInfos := map[string]*SystemInfo{}
 	for sysName, sysInfo := range pkg.Systems {
+
+		// guard against missing search data
+		if sysInfo.StoreHash == "" || sysInfo.StoreName == "" {
+			continue
+		}
+
+		storePath := nix.StorePath(sysInfo.StoreHash, sysInfo.StoreName, sysInfo.StoreVersion)
+		caStorePath := ""
+		if sysName == userSystem {
+			caStorePath, err = nix.ContentAddressedStorePath(storePath)
+			if err != nil {
+				return nil, err
+			}
+		}
 		sysInfos[sysName] = &SystemInfo{
-			System:       sysName,
-			FromHash:     sysInfo.StoreHash,
-			StoreName:    sysInfo.StoreName,
-			StoreVersion: sysInfo.StoreVersion,
+			StorePath:   storePath,
+			CAStorePath: caStorePath,
 		}
 	}
-	return sysInfos
+	return sysInfos, nil
 }

internal/nix/nix.go

@@ -66,9 +66,6 @@ func (*Nix) PrintDevEnv(ctx context.Context, args *PrintDevEnvArgs) (*PrintDevEn
 			args.FlakesFilePath,
 		)
 		cmd.Args = append(cmd.Args, ExperimentalFlags()...)
-		if featureflag.RemoveNixpkgs.Enabled() {
-			cmd.Args = append(cmd.Args, "--impure")
-		}
 		cmd.Args = append(cmd.Args, "--json")
 		debug.Log("Running print-dev-env cmd: %s\n", cmd)
 		data, err = cmd.Output()

internal/nix/store.go

@@ -0,0 +1,44 @@
+package nix
+
+import (
+	"encoding/json"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+func StorePath(hash, name, version string) string {
+	storeDirParts := []string{hash, name}
+	if version != "" {
+		storeDirParts = append(storeDirParts, version)
+	}
+	storeDir := strings.Join(storeDirParts, "-")
+	return filepath.Join("/nix/store", storeDir)
+}
+
+// ContentAddressedStorePath takes a store path and returns the content-addressed store path.
+func ContentAddressedStorePath(storePath string) (string, error) {
+	cmd := command("store", "make-content-addressed", storePath, "--json")
+	out, err := cmd.Output()
+	if err != nil {
+		return "", errors.WithStack(err)
+	}
+	// Example Output:
+	// > nix store make-content-addressed /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1 --json
+	// {"rewrites":{"/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1":"/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1"}}
+
+	type ContentAddressed struct {
+		Rewrites map[string]string `json:"rewrites"`
+	}
+	caOutput := ContentAddressed{}
+	if err := json.Unmarshal(out, &caOutput); err != nil {
+		return "", errors.WithStack(err)
+	}
+
+	caStorePath, ok := caOutput.Rewrites[storePath]
+	if !ok {
+		return "", errors.Errorf("could not find content-addressed store path for %s", storePath)
+	}
+	return caStorePath, nil
+}

internal/nix/store_test.go

@@ -0,0 +1,32 @@
+package nix
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestContentAddressedPath(t *testing.T) {
+
+	testCases := []struct {
+		storePath string
+		expected  string
+	}{
+		{
+			"/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1",
+			"/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1",
+		},
+	}
+
+	for index, testCase := range testCases {
+		t.Run(fmt.Sprintf("%d", index), func(t *testing.T) {
+			out, err := ContentAddressedStorePath(testCase.storePath)
+			if err != nil {
+				t.Errorf("got error: %v", err)
+			}
+			if out != testCase.expected {
+				t.Errorf("got %s, want %s", out, testCase.expected)
+			}
+		})
+
+	}
+}

internal/shellgen/tmpl/flake_remove_nixpkgs.nix.tmpl

@@ -34,6 +34,7 @@
             (builtins.fetchClosure{
               fromStore = "{{ $.BinaryCacheStore }}";
               fromPath = "{{ .PathInBinaryStore }}";
+              toPath = "{{ .ContentAddressedStorePath }}";
             })
             {{- end }}
             {{- end }}