[remove nixpkgs] Remove CAPath, and only enable for nix >= 2.17 (#1332)

## Summary

We want to restrict the feature to nix 2.17+, because it enables pure
mode nix
when using just `fromPath` in `builtins.fetchClosure`.

This greatly simplifies the `devbox.lock` issues with caPath, and will
improve the UX.

This Remove Nixpkgs feature is most beneficial to new users who have not
downloaded the nixpkgs they need. These users are most likely to have
the latest nix installed for them, so I think restricting to nix >= 2.17
is an acceptable tradeoff.

## How was it tested?

did a sanity test on x86-darwin.
~need to test better~ on a "supported system" where nix >= 2.17:
  - updated Devbox Cloud to run nix 2.17
- in `examples/development/python/pip`, did `devbox shell`. Since we
have system-info for `python` and not `pip`, the generated `flake.nix`
looked like this:
```
{
   description = "A devbox shell";

   inputs = {
     nixpkgs.url = "http://lax.devbox-nixed.internal/nixos/nixpkgs/archive/8670e496ffd093b60e74e7fa53526aa5920d09eb.tar.gz";
     nixpkgs-8670e4.url = "http://lax.devbox-nixed.internal/nixos/nixpkgs/archive/8670e496ffd093b60e74e7fa53526aa5920d09eb.tar.gz";
   };

   outputs = {
     self,
     nixpkgs,
     nixpkgs-8670e4,
   }:
      let
        pkgs = nixpkgs.legacyPackages.x86_64-linux;
        nixpkgs-8670e4-pkgs = (import nixpkgs-8670e4 {
          system = "x86_64-linux";
          config.allowUnfree = true;
          config.permittedInsecurePackages = [
          ];
        });
      in
      {
        devShells.x86_64-linux.default = pkgs.mkShell {
          buildInputs = [
            (builtins.fetchClosure{
              fromStore = "https://cache.nixos.org";
              fromPath = "/nix/store/95cxzy2hpizr23343b8bskl4yacf4b3l-python3-3.10.11";
              inputAddressed = true;
            })
            nixpkgs-8670e4-pkgs.python310Packages.pip
          ];
        };
      };
 }

```

Author: savil

internal/devpkg/package.go

@@ -9,7 +9,6 @@ import (
 	"fmt"
 	"io"
 	"net/url"
-	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -21,7 +20,7 @@ import (
 	"go.jetpack.io/devbox/internal/cuecfg"
 	"go.jetpack.io/devbox/internal/lock"
 	"go.jetpack.io/devbox/internal/nix"
-	"go.jetpack.io/devbox/internal/ux"
+	"go.jetpack.io/devbox/internal/vercheck"
 	"go.jetpack.io/devbox/plugins"
 )
 
@@ -156,7 +155,7 @@ func (p *Package) Installable() (string, error) {
 	}
 
 	if inCache {
-		installable, err := p.ContentAddressedPath()
+		installable, err := p.InputAddressedPath()
 		if err != nil {
 			return "", err
 		}
@@ -466,36 +465,22 @@ func (p *Package) IsInBinaryCache() (bool, error) {
 
 	// Check if the user's system's info is present in the lockfile
 	_, ok := entry.Systems[userSystem]
-	return ok, nil
-}
-
-// InputAddressedPath is the input-addressed path in /nix/store
-// It is also the key in the BinaryCache for this package
-func (p *Package) InputAddressedPath() (string, error) {
-	if inCache, err := p.IsInBinaryCache(); err != nil {
-		return "", err
-	} else if !inCache {
-		return "",
-			errors.Errorf("Package %q cannot be fetched from binary cache store", p.Raw)
+	if !ok {
+		return false, nil
 	}
 
-	entry, err := p.lockfile.Resolve(p.Raw)
+	version, err := nix.Version()
 	if err != nil {
-		return "", err
-	}
-
-	userSystem, err := nix.System()
-	if err != nil {
-		return "", err
+		return false, err
 	}
 
-	sysInfo := entry.Systems[userSystem]
-	return sysInfo.StorePath, nil
+	// enable for nix >= 2.17
+	return vercheck.SemverCompare(version, "2.17.0") >= 0, nil
 }
 
-// ContentAddressedPath is the content-addressed form of Package.InputAddressedPath
-func (p *Package) ContentAddressedPath() (string, error) {
-
+// InputAddressedPath is the input-addressed path in /nix/store
+// It is also the key in the BinaryCache for this package
+func (p *Package) InputAddressedPath() (string, error) {
 	if inCache, err := p.IsInBinaryCache(); err != nil {
 		return "", err
 	} else if !inCache {
@@ -514,20 +499,7 @@ func (p *Package) ContentAddressedPath() (string, error) {
 	}
 
 	sysInfo := entry.Systems[userSystem]
-	if sysInfo.CAStorePath != "" {
-		return sysInfo.CAStorePath, nil
-	}
-
-	ux.Fwarning(
-		os.Stderr,
-		fmt.Sprintf("calculating ca_store_path for %s. This may be slow. "+
-			"Run `devbox update` to speed this up for next time.\n", sysInfo.StorePath),
-	)
-	localPath, err := nix.ContentAddressedStorePath(sysInfo.StorePath)
-	if err != nil {
-		return "", err
-	}
-	return localPath, err
+	return sysInfo.StorePath, nil
 }
 
 func (p *Package) AllowInsecure() bool {

internal/impl/update_test.go

@@ -44,8 +44,7 @@ func TestUpdateNewCurrentSysInfoIsAdded(t *testing.T) {
 		Resolved: "resolved-flake-reference",
 		Systems: map[string]*lock.SystemInfo{
 			sys: {
-				StorePath:   "store_path1",
-				CAStorePath: "ca_path1",
+				StorePath: "store_path1",
 			},
 		},
 	}
@@ -64,7 +63,6 @@ func TestUpdateNewCurrentSysInfoIsAdded(t *testing.T) {
 	require.Contains(t, lockfile.Packages, raw)
 	require.Contains(t, lockfile.Packages[raw].Systems, sys)
 	require.Equal(t, "store_path1", lockfile.Packages[raw].Systems[sys].StorePath)
-	require.Equal(t, "ca_path1", lockfile.Packages[raw].Systems[sys].CAStorePath)
 }
 
 func TestUpdateNewSysInfoIsAdded(t *testing.T) {
@@ -81,8 +79,7 @@ func TestUpdateNewSysInfoIsAdded(t *testing.T) {
 		Resolved: "resolved-flake-reference",
 		Systems: map[string]*lock.SystemInfo{
 			sys1: {
-				StorePath:   "store_path1",
-				CAStorePath: "ca_path1",
+				StorePath: "store_path1",
 			},
 			sys2: {
 				StorePath: "store_path2",
@@ -95,8 +92,7 @@ func TestUpdateNewSysInfoIsAdded(t *testing.T) {
 				Resolved: "resolved-flake-reference",
 				Systems: map[string]*lock.SystemInfo{
 					sys1: {
-						StorePath:   "store_path1",
-						CAStorePath: "ca_path1",
+						StorePath: "store_path1",
 					},
 					// Missing sys2
 				},
@@ -127,8 +123,7 @@ func TestUpdateOtherSysInfoIsReplaced(t *testing.T) {
 		Resolved: "resolved-flake-reference",
 		Systems: map[string]*lock.SystemInfo{
 			sys1: {
-				StorePath:   "store_path1",
-				CAStorePath: "ca_path1",
+				StorePath: "store_path1",
 			},
 			sys2: {
 				StorePath: "store_path2",
@@ -141,12 +136,10 @@ func TestUpdateOtherSysInfoIsReplaced(t *testing.T) {
 				Resolved: "resolved-flake-reference",
 				Systems: map[string]*lock.SystemInfo{
 					sys1: {
-						StorePath:   "store_path1",
-						CAStorePath: "ca_path1",
+						StorePath: "store_path1",
 					},
 					sys2: {
-						StorePath:   "mismatching_store_path",
-						CAStorePath: "ca_path2",
+						StorePath: "mismatching_store_path",
 					},
 				},
 			},
@@ -161,58 +154,6 @@ func TestUpdateOtherSysInfoIsReplaced(t *testing.T) {
 	require.Contains(t, lockfile.Packages[raw].Systems, sys2)
 	require.Equal(t, "store_path1", lockfile.Packages[raw].Systems[sys1].StorePath)
 	require.Equal(t, "store_path2", lockfile.Packages[raw].Systems[sys2].StorePath)
-	require.Empty(t, lockfile.Packages[raw].Systems[sys2].CAStorePath)
-}
-
-func TestUpdateCAPathIsNotReplaced(t *testing.T) {
-	featureflag.RemoveNixpkgs.EnableForTest(t)
-	devbox := devboxForTesting(t)
-
-	raw := "[email protected]"
-	sys1 := currentSystem(t)
-	sys2 := "system2"
-	devPkg := &devpkg.Package{
-		Raw: raw,
-	}
-	resolved := &lock.Package{
-		Resolved: "resolved-flake-reference",
-		Systems: map[string]*lock.SystemInfo{
-			sys1: {
-				StorePath:   "store_path1",
-				CAStorePath: "ca_path1",
-			},
-			sys2: {
-				StorePath: "store_path2",
-				// No CAPath here because this is not the current system.
-			},
-		},
-	}
-	lockfile := &lock.File{
-		Packages: map[string]*lock.Package{
-			raw: {
-				Resolved: "resolved-flake-reference",
-				Systems: map[string]*lock.SystemInfo{
-					sys1: {
-						StorePath:   "store_path1",
-						CAStorePath: "ca_path1",
-					},
-					sys2: {
-						StorePath:   "store_path2",
-						CAStorePath: "ca_path2", // we already have CAPath for this system; it should not be replaced
-					},
-				},
-			},
-		},
-	}
-
-	err := devbox.mergeResolvedPackageToLockfile(context.Background(), devPkg, resolved, lockfile)
-	require.NoError(t, err, "update failed")
-
-	require.Contains(t, lockfile.Packages, raw)
-	require.Contains(t, lockfile.Packages[raw].Systems, sys1)
-	require.Contains(t, lockfile.Packages[raw].Systems, sys2)
-	require.Equal(t, "store_path2", lockfile.Packages[raw].Systems[sys2].StorePath)
-	require.Equal(t, "ca_path2", lockfile.Packages[raw].Systems[sys2].CAStorePath)
 }
 
 func currentSystem(t *testing.T) string {

internal/lock/package.go

@@ -25,9 +25,6 @@ type SystemInfo struct {
 	// It is of the form /nix/store/<hash>-<name>-<version>
 	// <name> may be different from the canonicalName so we store the full store path.
 	StorePath string `json:"store_path,omitempty"`
-	// CAStorePath is the content-addressed path for the nix package in /nix/store
-	// It is of the form /nix/store/<hash>-<name>-<version>
-	CAStorePath string `json:"ca_store_path,omitempty"`
 }
 
 func (p *Package) GetSource() string {

internal/lock/resolve.go

@@ -35,10 +35,7 @@ func (f *File) FetchResolvedPackage(pkg string) (*Package, error) {
 
 	sysInfos := map[string]*SystemInfo{}
 	if featureflag.RemoveNixpkgs.Enabled() {
-		sysInfos, err = buildLockSystemInfos(packageVersion)
-		if err != nil {
-			return nil, err
-		}
+		sysInfos = buildLockSystemInfos(packageVersion)
 	}
 	packageInfo, err := selectForSystem(packageVersion)
 	if err != nil {
@@ -81,12 +78,7 @@ func selectForSystem(pkg *searcher.PackageVersion) (searcher.PackageInfo, error)
 	return maps.Values(pkg.Systems)[0], nil
 }
 
-func buildLockSystemInfos(pkg *searcher.PackageVersion) (map[string]*SystemInfo, error) {
-	userSystem, err := nix.System()
-	if err != nil {
-		return nil, err
-	}
-
+func buildLockSystemInfos(pkg *searcher.PackageVersion) map[string]*SystemInfo {
 	sysInfos := map[string]*SystemInfo{}
 	for sysName, sysInfo := range pkg.Systems {
 
@@ -97,17 +89,9 @@ func buildLockSystemInfos(pkg *searcher.PackageVersion) (map[string]*SystemInfo,
 		}
 
 		storePath := nix.StorePath(sysInfo.StoreHash, sysInfo.StoreName, sysInfo.StoreVersion)
-		caStorePath := ""
-		if sysName == userSystem {
-			caStorePath, err = nix.ContentAddressedStorePath(storePath)
-			if err != nil {
-				return nil, errors.WithMessagef(err, "failed to make content addressed path for %s", storePath)
-			}
-		}
 		sysInfos[sysName] = &SystemInfo{
-			StorePath:   storePath,
-			CAStorePath: caStorePath,
+			StorePath: storePath,
 		}
 	}
-	return sysInfos, nil
+	return sysInfos
 }

internal/nix/nix.go

@@ -144,6 +144,31 @@ func System() (string, error) {
 	return cachedSystem, nil
 }
 
+// version is the cached output of `nix --version`.
+var version = ""
+
+// Version returns the version of nix from `nix --version`. Usually in a semver
+// like format, but not strictly.
+func Version() (string, error) {
+
+	if version != "" {
+		return version, nil
+	}
+
+	cmd := command("--version")
+	outBytes, err := cmd.Output()
+	if err != nil {
+		return "", errors.WithStack(err)
+	}
+	out := string(outBytes)
+	const prefix = "nix (Nix) "
+	if !strings.HasPrefix(out, prefix) {
+		return "", errors.Errorf(`Expected "%s" prefix, but output from nix --version was: %s`, prefix, out)
+	}
+	version = strings.TrimSpace(strings.TrimPrefix(out, prefix))
+	return version, nil
+}
+
 // Warning: be careful using the bins in default/bin, they won't always match bins
 // produced by the flakes.nix. Use devbox.NixBins() instead.
 func ProfileBinPath(projectDir string) string {

internal/nix/store.go

@@ -1,11 +1,8 @@
 package nix
 
 import (
-	"encoding/json"
 	"path/filepath"
 	"strings"
-
-	"github.com/pkg/errors"
 )
 
 func StorePath(hash, name, version string) string {
@@ -16,29 +13,3 @@ func StorePath(hash, name, version string) string {
 	storeDir := strings.Join(storeDirParts, "-")
 	return filepath.Join("/nix/store", storeDir)
 }
-
-// ContentAddressedStorePath takes a store path and returns the content-addressed store path.
-func ContentAddressedStorePath(storePath string) (string, error) {
-	cmd := command("store", "make-content-addressed", storePath, "--json")
-	out, err := cmd.Output()
-	if err != nil {
-		return "", errors.WithStack(err)
-	}
-	// Example Output:
-	// > nix store make-content-addressed /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1 --json
-	// {"rewrites":{"/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1":"/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1"}}
-
-	type ContentAddressed struct {
-		Rewrites map[string]string `json:"rewrites"`
-	}
-	caOutput := ContentAddressed{}
-	if err := json.Unmarshal(out, &caOutput); err != nil {
-		return "", errors.WithStack(err)
-	}
-
-	caStorePath, ok := caOutput.Rewrites[storePath]
-	if !ok {
-		return "", errors.Errorf("could not find content-addressed store path for %s", storePath)
-	}
-	return caStorePath, nil
-}

internal/nix/store_test.go

@@ -1,39 +1 @@
 package nix
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-
-	"golang.org/x/exp/slices"
-)
-
-func TestContentAddressedPath(t *testing.T) {
-	testCases := []struct {
-		storePath string
-		expected  []string
-	}{
-		{
-			"/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1",
-			[]string{
-				// Hash from before Nix 2.17.0.
-				"/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1",
-
-				// Hash after Nix 2.17.0.
-				"/nix/store/d49wyvsz5nkqa23qp4p0ikr04mw9n4h9-git-2.33.1",
-			},
-		},
-	}
-
-	for index, testCase := range testCases {
-		t.Run(fmt.Sprintf("%d", index), func(t *testing.T) {
-			out, err := ContentAddressedStorePath(testCase.storePath)
-			if err != nil {
-				t.Errorf("got error: %v", err)
-			}
-			if !slices.Contains(testCase.expected, out) {
-				t.Errorf("got %q, want any of:\n%s", out, strings.Join(testCase.expected, "\n"))
-			}
-		})
-	}
-}

internal/shellgen/tmpl/flake_remove_nixpkgs.nix.tmpl

@@ -41,7 +41,7 @@
             (builtins.fetchClosure{
               fromStore = "{{ $.BinaryCache }}";
               fromPath = "{{ .InputAddressedPath }}";
-              toPath = "{{ .ContentAddressedPath }}";
+              inputAddressed = true;
             })
             {{- end }}
             {{- end }}