[cache] Optimize cache upload (#2083)

## Summary

This changes `devbox cache upload` so that it now longer has to build
packages if they are already in cache.

For each package, it first checks if it is in target cache. If it is, it
skips. Otherwise it builds/copies.

## How was it tested?

<img width="843" alt="image"
src="https://github.com/jetify-com/devbox/assets/544948/225b744a-ff20-446e-a7d0-6436005a33ac">

Author: mikeland73

internal/devbox/cache.go

@@ -5,10 +5,13 @@ import (
 	"errors"
 	"io"
 
+	"github.com/samber/lo"
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/build"
+	"go.jetpack.io/devbox/internal/debug"
 	"go.jetpack.io/devbox/internal/devbox/providers/identity"
 	"go.jetpack.io/devbox/internal/devbox/providers/nixcache"
+	"go.jetpack.io/devbox/internal/devpkg"
 	"go.jetpack.io/devbox/internal/nix"
 	"go.jetpack.io/devbox/internal/ux"
 	"go.jetpack.io/pkg/auth"
@@ -18,6 +21,7 @@ func (d *Devbox) UploadProjectToCache(
 	ctx context.Context,
 	cacheURI string,
 ) error {
+	defer debug.FunctionTimer().End()
 	if cacheURI == "" {
 		var err error
 		cacheURI, err = getWriteCacheURI(ctx, d.stderr)
@@ -30,19 +34,35 @@ func (d *Devbox) UploadProjectToCache(
 	if err != nil && !errors.Is(err, auth.ErrNotLoggedIn) {
 		return err
 	}
-	profilePath, err := d.profilePath()
-	if err != nil {
+
+	packages := lo.Filter(d.InstallablePackages(), devpkg.IsNix)
+	if err != nil || len(packages) == 0 {
 		return err
 	}
 
-	// Ensure state is up to date before uploading to cache.
-	// TODO: we may be able to do this more efficiently, not sure everything needs
-	// to be installed.
-	if err = d.ensureStateIsUpToDate(ctx, ensure); err != nil {
-		return err
+	for _, pkg := range packages {
+		inCache, err := pkg.AreAllOutputsInCache(ctx, d.stderr, cacheURI)
+		if err != nil {
+			return err
+		}
+		if inCache {
+			ux.Finfo(d.stderr, "Package %s is already in cache, skipping\n", pkg.Raw)
+			continue
+		}
+		ux.Finfo(d.stderr, "Uploading package %s to cache\n", pkg.Raw)
+		installables, err := pkg.Installables()
+		if err != nil {
+			return err
+		}
+		for _, installable := range installables {
+			err := nix.CopyInstallableToCache(ctx, d.stderr, cacheURI, installable, creds.Env())
+			if err != nil {
+				return err
+			}
+		}
 	}
 
-	return nix.CopyInstallableToCache(ctx, d.stderr, cacheURI, profilePath, creds.Env())
+	return nil
 }
 
 func UploadInstallableToCache(

internal/devbox/packages.go

@@ -553,32 +553,15 @@ func (d *Devbox) packagesToInstallInStore(ctx context.Context, mode installMode)
 	packagesToInstall := []*devpkg.Package{}
 	storePathsForPackage := map[*devpkg.Package][]string{}
 	for _, pkg := range packages {
-		installables, err := pkg.Installables()
+		if mode == update {
+			packagesToInstall = append(packagesToInstall, pkg)
+			continue
+		}
+		var err error
+		storePathsForPackage[pkg], err = pkg.GetStorePaths(ctx, d.stderr)
 		if err != nil {
 			return nil, err
 		}
-		for _, installable := range installables {
-			if mode == update {
-				packagesToInstall = append(packagesToInstall, pkg)
-				continue
-			}
-
-			resolvedStorePaths, err := pkg.GetResolvedStorePaths()
-			if err != nil {
-				return nil, err
-			}
-			if len(resolvedStorePaths) > 0 {
-				storePathsForPackage[pkg] = append(storePathsForPackage[pkg], resolvedStorePaths...)
-				continue
-			}
-
-			storePathsForInstallable, err := nix.StorePathsFromInstallable(
-				ctx, installable, pkg.HasAllowInsecure())
-			if err != nil {
-				return nil, packageInstallErrorHandler(err, pkg, installable)
-			}
-			storePathsForPackage[pkg] = append(storePathsForPackage[pkg], storePathsForInstallable...)
-		}
 	}
 
 	// Batch this for perf
@@ -599,58 +582,6 @@ func (d *Devbox) packagesToInstallInStore(ctx context.Context, mode installMode)
 	return lo.Uniq(packagesToInstall), nil
 }
 
-// packageInstallErrorHandler checks for two kinds of errors to print custom messages for so that Devbox users
-// can work around them:
-// 1. Packages that cannot be installed on the current system, but may be installable on other systems.packageInstallErrorHandler
-// 2. Packages marked insecure by nix
-func packageInstallErrorHandler(err error, pkg *devpkg.Package, installableOrEmpty string) error {
-	if err == nil {
-		return nil
-	}
-
-	// Check if the user is installing a package that cannot be installed on their platform.
-	// For example, glibcLocales on MacOS will give the following error:
-	// flake output attribute 'legacyPackages.x86_64-darwin.glibcLocales' is not a derivation or path
-	// This is because glibcLocales is only available on Linux.
-	// The user should try `devbox add` again with `--exclude-platform`
-	errMessage := strings.TrimSpace(err.Error())
-
-	// Sample error from `devbox add glibcLocales` on a mac:
-	// error: flake output attribute 'legacyPackages.x86_64-darwin.glibcLocales' is not a derivation or path
-	maybePackageSystemCompatibilityErrorType1 := strings.Contains(errMessage, "error: flake output attribute") &&
-		strings.Contains(errMessage, "is not a derivation or path")
-	// Sample error from `devbox add sublime4` on a mac:
-	// error: Package ‘sublimetext4-4169’ in /nix/store/nlbjx0mp83p2qzf1rkmzbgvq1wxfir81-source/pkgs/applications/editors/sublime/4/common.nix:168 is not available on the requested hostPlatform:
-	//     hostPlatform.config = "x86_64-apple-darwin"
-	//     package.meta.platforms = [
-	//       "aarch64-linux"
-	//       "x86_64-linux"
-	//    ]
-	maybePackageSystemCompatibilityErrorType2 := strings.Contains(errMessage, "is not available on the requested hostPlatform")
-
-	if maybePackageSystemCompatibilityErrorType1 || maybePackageSystemCompatibilityErrorType2 {
-		platform := nix.System()
-		return usererr.WithUserMessage(
-			err,
-			"package %s cannot be installed on your platform %s.\n"+
-				"If you know this package is incompatible with %[2]s, then "+
-				"you could run `devbox add %[1]s --exclude-platform %[2]s` and re-try.\n"+
-				"If you think this package should be compatible with %[2]s, then "+
-				"it's possible this particular version is not available yet from the nix registry. "+
-				"You could try `devbox add` with a different version for this package.\n\n"+
-				"Underlying Error from nix is:",
-			pkg.Versioned(),
-			platform,
-		)
-	}
-
-	if isInsecureErr, userErr := nix.IsExitErrorInsecurePackage(err, pkg.Versioned(), installableOrEmpty); isInsecureErr {
-		return userErr
-	}
-
-	return usererr.WithUserMessage(err, "error installing package %s", pkg.Raw)
-}
-
 // moveAllowInsecureFromLockfile will modernize a Devbox project by moving the allow_insecure: boolean
 // setting from the devbox.lock file to the corresponding package in devbox.json.
 //

internal/devpkg/narinfo_cache.go

@@ -3,6 +3,7 @@ package devpkg
 import (
 	"context"
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
@@ -172,6 +173,32 @@ func (p *Package) fetchNarInfoStatusOnce(
 	return outputToCache, nil
 }
 
+func (p *Package) AreAllOutputsInCache(
+	ctx context.Context, w io.Writer, cacheURI string,
+) (bool, error) {
+	storePaths, err := p.GetStorePaths(ctx, w)
+	if err != nil {
+		return false, err
+	}
+
+	for _, storePath := range storePaths {
+		pathParts := nix.NewStorePathParts(storePath)
+		hash := pathParts.Hash
+		if strings.HasPrefix(cacheURI, "s3") {
+			inCache, err := fetchNarInfoStatusFromS3(ctx, cacheURI, hash)
+			if err != nil || !inCache {
+				return false, err
+			}
+		} else {
+			inCache, err := fetchNarInfoStatusFromHTTP(ctx, cacheURI, hash)
+			if err != nil || !inCache {
+				return false, err
+			}
+		}
+	}
+	return true, nil
+}
+
 func (p *Package) outputsForOutputName(output string) ([]lock.Output, error) {
 	sysInfo, err := p.sysInfoIfExists()
 	if err != nil || sysInfo == nil {

internal/devpkg/package.go

@@ -22,6 +22,7 @@ import (
 	"go.jetpack.io/devbox/internal/devpkg/pkgtype"
 	"go.jetpack.io/devbox/internal/lock"
 	"go.jetpack.io/devbox/internal/nix"
+	"go.jetpack.io/devbox/internal/ux"
 	"go.jetpack.io/devbox/nix/flake"
 	"go.jetpack.io/devbox/plugins"
 )
@@ -734,3 +735,84 @@ func (p *Package) GetResolvedStorePaths() ([]string, error) {
 	}
 	return storePaths, nil
 }
+
+func (p *Package) GetStorePaths(ctx context.Context, w io.Writer) ([]string, error) {
+	storePathsForPackage, err := p.GetResolvedStorePaths()
+	if err != nil || len(storePathsForPackage) > 0 {
+		return storePathsForPackage, err
+	}
+
+	// No fast path, we need to query nix.
+	// TODO we should give people the option to add paths to lockfile.
+	ux.Fwarning(
+		w,
+		"Outputs for %s are not in lockfile. Fetching store paths from nix, this may take a while\n",
+		p.Raw,
+	)
+
+	installables, err := p.Installables()
+	if err != nil {
+		return nil, err
+	}
+	for _, installable := range installables {
+		storePathsForInstallable, err := nix.StorePathsFromInstallable(
+			ctx, installable, p.HasAllowInsecure())
+		if err != nil {
+			return nil, packageInstallErrorHandler(err, p, installable)
+		}
+		storePathsForPackage = append(storePathsForPackage, storePathsForInstallable...)
+	}
+	return storePathsForPackage, nil
+}
+
+// packageInstallErrorHandler checks for two kinds of errors to print custom messages for so that Devbox users
+// can work around them:
+// 1. Packages that cannot be installed on the current system, but may be installable on other systems.packageInstallErrorHandler
+// 2. Packages marked insecure by nix
+func packageInstallErrorHandler(err error, pkg *Package, installableOrEmpty string) error {
+	if err == nil {
+		return nil
+	}
+
+	// Check if the user is installing a package that cannot be installed on their platform.
+	// For example, glibcLocales on MacOS will give the following error:
+	// flake output attribute 'legacyPackages.x86_64-darwin.glibcLocales' is not a derivation or path
+	// This is because glibcLocales is only available on Linux.
+	// The user should try `devbox add` again with `--exclude-platform`
+	errMessage := strings.TrimSpace(err.Error())
+
+	// Sample error from `devbox add glibcLocales` on a mac:
+	// error: flake output attribute 'legacyPackages.x86_64-darwin.glibcLocales' is not a derivation or path
+	maybePackageSystemCompatibilityErrorType1 := strings.Contains(errMessage, "error: flake output attribute") &&
+		strings.Contains(errMessage, "is not a derivation or path")
+	// Sample error from `devbox add sublime4` on a mac:
+	// error: Package ‘sublimetext4-4169’ in /nix/store/nlbjx0mp83p2qzf1rkmzbgvq1wxfir81-source/pkgs/applications/editors/sublime/4/common.nix:168 is not available on the requested hostPlatform:
+	//     hostPlatform.config = "x86_64-apple-darwin"
+	//     package.meta.platforms = [
+	//       "aarch64-linux"
+	//       "x86_64-linux"
+	//    ]
+	maybePackageSystemCompatibilityErrorType2 := strings.Contains(errMessage, "is not available on the requested hostPlatform")
+
+	if maybePackageSystemCompatibilityErrorType1 || maybePackageSystemCompatibilityErrorType2 {
+		platform := nix.System()
+		return usererr.WithUserMessage(
+			err,
+			"package %s cannot be installed on your platform %s.\n"+
+				"If you know this package is incompatible with %[2]s, then "+
+				"you could run `devbox add %[1]s --exclude-platform %[2]s` and re-try.\n"+
+				"If you think this package should be compatible with %[2]s, then "+
+				"it's possible this particular version is not available yet from the nix registry. "+
+				"You could try `devbox add` with a different version for this package.\n\n"+
+				"Underlying Error from nix is:",
+			pkg.Versioned(),
+			platform,
+		)
+	}
+
+	if isInsecureErr, userErr := nix.IsExitErrorInsecurePackage(err, pkg.Versioned(), installableOrEmpty); isInsecureErr {
+		return userErr
+	}
+
+	return usererr.WithUserMessage(err, "error installing package %s", pkg.Raw)
+}