[cache] Use jetify caches (#2072)

## Summary

The main change this PR does is that it considers jetify read caches
when determining if a given store path is in binary cache. It loops
though all caches (default nix cache, jetify provided caches) and if it
finds a path in any one of them, we know output is in binary cache. It
also changes the fetch nar info function so that it stores which cache
each output is in since we may now use multiple caches.

List of changes:

* `fetchNarInfoStatusOnce` now checks all caches and returns which cache
each output is in.
* Moved in memory cache to be at the request level instead of output
level. This is more efficient because the meta-output
`__default_output__ ` can now propertly use the request cache.
* Changed providers to be true singletons.
* Small improvements to flake template to prepare for each output using
own cache. (Currently there's a hack in there where we always use nix
cache as `fromStore` even if not correct. See comment in code.

## How was it tested?

Installed `https://github.com/Lagoja/terraform-test` and confirmed cache
was used.

Author: mikeland73

internal/boxcli/auth.go

@@ -38,7 +38,7 @@ func loginCmd() *cobra.Command {
 		Short: "Login to devbox",
 		Args:  cobra.ExactArgs(0),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			c, err := identity.Get().AuthClient()
+			c, err := identity.AuthClient()
 			if err != nil {
 				return err
 			}
@@ -62,7 +62,7 @@ func logoutCmd() *cobra.Command {
 		Short: "Logout from devbox",
 		Args:  cobra.ExactArgs(0),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			c, err := identity.Get().AuthClient()
+			c, err := identity.AuthClient()
 			if err != nil {
 				return err
 			}
@@ -123,7 +123,7 @@ func authNewTokenCommand() *cobra.Command {
 		Args:  cobra.ExactArgs(0),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
-			token, err := identity.Get().GenSession(ctx)
+			token, err := identity.GenSession(ctx)
 			if err != nil {
 				return err
 			}

internal/boxcli/cache.go

@@ -89,7 +89,7 @@ func cacheConfigureCmd() *cobra.Command {
 				u, _ := user.Current()
 				username = u.Username
 			}
-			return nixcache.Get().ConfigureReprompt(cmd.Context(), username)
+			return nixcache.ConfigureReprompt(cmd.Context(), username)
 		},
 	}
 	cmd.Flags().StringVar(&username, "user", "", "")
@@ -104,7 +104,7 @@ func cacheCredentialsCmd() *cobra.Command {
 		Hidden: true,
 		Args:   cobra.ExactArgs(0),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			creds, err := nixcache.Get().Credentials(cmd.Context())
+			creds, err := nixcache.CachedCredentials(cmd.Context())
 			if err != nil {
 				return err
 			}
@@ -136,7 +136,7 @@ func cacheInfoCmd() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			// TODO(gcurtis): We can also output info about the daemon config status
 			// here
-			caches, err := nixcache.Get().Caches(cmd.Context())
+			caches, err := nixcache.Caches(cmd.Context())
 			if err != nil {
 				return err
 			}

internal/boxcli/pull.go

@@ -65,7 +65,7 @@ func pullCmdFunc(cmd *cobra.Command, url string, flags *pullCmdFlags) error {
 	}
 
 	var creds devopt.Credentials
-	t, err := identity.Get().GenSession(cmd.Context())
+	t, err := identity.GenSession(cmd.Context())
 	if err != nil && !errors.Is(err, auth.ErrNotLoggedIn) {
 		return errors.WithStack(err)
 	} else if t != nil && err == nil {

internal/boxcli/push.go

@@ -44,7 +44,7 @@ func pushCmdFunc(cmd *cobra.Command, url string, flags pushCmdFlags) error {
 	if err != nil {
 		return errors.WithStack(err)
 	}
-	t, err := identity.Get().GenSession(cmd.Context())
+	t, err := identity.GenSession(cmd.Context())
 	var creds devopt.Credentials
 	if err != nil && !errors.Is(err, auth.ErrNotLoggedIn) {
 		return errors.WithStack(err)

internal/devbox/cache.go

@@ -6,6 +6,7 @@ import (
 	"io"
 
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
+	"go.jetpack.io/devbox/internal/devbox/providers/identity"
 	"go.jetpack.io/devbox/internal/devbox/providers/nixcache"
 	"go.jetpack.io/devbox/internal/nix"
 	"go.jetpack.io/devbox/internal/ux"
@@ -18,13 +19,13 @@ func (d *Devbox) UploadProjectToCache(
 ) error {
 	if cacheURI == "" {
 		var err error
-		cacheURI, err = getWriteCacheURI(ctx, d.stderr, d.providers.NixCache)
+		cacheURI, err = getWriteCacheURI(ctx, d.stderr)
 		if err != nil {
 			return err
 		}
 	}
 
-	creds, err := d.providers.NixCache.Credentials(ctx)
+	creds, err := nixcache.CachedCredentials(ctx)
 	if err != nil && !errors.Is(err, auth.ErrNotLoggedIn) {
 		return err
 	}
@@ -50,13 +51,13 @@ func UploadInstallableToCache(
 ) error {
 	if cacheURI == "" {
 		var err error
-		cacheURI, err = getWriteCacheURI(ctx, stderr, *nixcache.Get())
+		cacheURI, err = getWriteCacheURI(ctx, stderr)
 		if err != nil {
 			return err
 		}
 	}
 
-	creds, err := nixcache.Get().Credentials(ctx)
+	creds, err := nixcache.CachedCredentials(ctx)
 	if err != nil && !errors.Is(err, auth.ErrNotLoggedIn) {
 		return err
 	}
@@ -66,9 +67,13 @@ func UploadInstallableToCache(
 func getWriteCacheURI(
 	ctx context.Context,
 	w io.Writer,
-	provider nixcache.Provider,
 ) (string, error) {
-	caches, err := provider.WriteCaches(ctx)
+	_, err := identity.GenSession(ctx)
+	if errors.Is(err, auth.ErrNotLoggedIn) {
+		return "",
+			usererr.New("You must be logged in to upload to a Nix cache.")
+	}
+	caches, err := nixcache.WriteCaches(ctx)
 	if err != nil {
 		return "", err
 	}

internal/devbox/devbox.go

@@ -27,7 +27,6 @@ import (
 	"go.jetpack.io/devbox/internal/cachehash"
 	"go.jetpack.io/devbox/internal/devbox/envpath"
 	"go.jetpack.io/devbox/internal/devbox/generate"
-	"go.jetpack.io/devbox/internal/devbox/providers"
 	"go.jetpack.io/devbox/internal/devpkg"
 	"go.jetpack.io/devbox/internal/devpkg/pkgtype"
 	"go.jetpack.io/devbox/internal/searcher"
@@ -66,7 +65,6 @@ type Devbox struct {
 	lockfile                 *lock.File
 	nix                      nix.Nixer
 	projectDir               string
-	providers                providers.Providers
 	pluginManager            *plugin.Manager
 	preservePathStack        bool
 	pure                     bool

internal/devbox/packages.go

@@ -20,6 +20,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/samber/lo"
 	"go.jetpack.io/devbox/internal/devbox/devopt"
+	"go.jetpack.io/devbox/internal/devbox/providers/nixcache"
 	"go.jetpack.io/devbox/internal/devconfig"
 	"go.jetpack.io/devbox/internal/devconfig/configfile"
 	"go.jetpack.io/devbox/internal/devpkg"
@@ -447,7 +448,7 @@ func (d *Devbox) installNixPackagesToStore(ctx context.Context, mode installMode
 		Flags:  flags,
 		Writer: d.stderr,
 	}
-	caches, err := d.providers.NixCache.ReadCaches(ctx)
+	caches, err := nixcache.CachedReadCaches(ctx)
 	if err != nil {
 		debug.Log("error getting nix cache URI, assuming user doesn't have access: %v", err)
 	}
@@ -460,7 +461,7 @@ func (d *Devbox) installNixPackagesToStore(ctx context.Context, mode installMode
 	// TODO (Landau): handle errors that are not auth.ErrNotLoggedIn
 	// Only lookup credentials if we have a cache to use
 	if len(args.ExtraSubstituters) > 0 {
-		creds, err := d.providers.NixCache.Credentials(ctx)
+		creds, err := nixcache.CachedCredentials(ctx)
 		if err == nil {
 			args.Env = creds.Env()
 		}
@@ -470,7 +471,7 @@ func (d *Devbox) installNixPackagesToStore(ctx context.Context, mode installMode
 			err = redact.Errorf("lookup current user: %v", err)
 			debug.Log("error configuring cache: %v", err)
 		}
-		err = d.providers.NixCache.Configure(ctx, u.Username)
+		err = nixcache.Configure(ctx, u.Username)
 		if err != nil {
 			debug.Log("error configuring cache: %v", err)
 

internal/devbox/providers/doc.go

@@ -0,0 +1,6 @@
+// providers encapsulates data and/or logic that can affect devbox behavior.
+// What a provider does can be influenced by the environment or external services.
+// The goal is to centralize this logic and avoid conditionals in core devbox code.
+// In the future we should allow dynamic providers as well which can help
+// customize devbox behavior.
+package providers

internal/devbox/providers/identity/identity.go

@@ -15,34 +15,26 @@ import (
 
 var scopes = []string{"openid", "offline_access", "email", "profile"}
 
-type Provider struct {
-	cachedAccessTokenFromAPIToken *session.Token
-}
-
-var singleton *Provider = &Provider{}
-
-func Get() *Provider {
-	return singleton
-}
+var cachedAccessTokenFromAPIToken *session.Token
 
-func (p *Provider) GenSession(ctx context.Context) (*session.Token, error) {
-	if t, err := p.getAccessTokenFromAPIToken(ctx); err != nil || t != nil {
+func GenSession(ctx context.Context) (*session.Token, error) {
+	if t, err := getAccessTokenFromAPIToken(ctx); err != nil || t != nil {
 		return t, err
 	}
 
-	c, err := p.AuthClient()
+	c, err := AuthClient()
 	if err != nil {
 		return nil, err
 	}
 	return c.GetSession(ctx)
 }
 
-func (p *Provider) Peek() (*session.Token, error) {
-	if p.cachedAccessTokenFromAPIToken != nil {
-		return p.cachedAccessTokenFromAPIToken, nil
+func Peek() (*session.Token, error) {
+	if cachedAccessTokenFromAPIToken != nil {
+		return cachedAccessTokenFromAPIToken, nil
 	}
 
-	c, err := p.AuthClient()
+	c, err := AuthClient()
 	if err != nil {
 		return nil, err
 	}
@@ -58,7 +50,7 @@ func (p *Provider) Peek() (*session.Token, error) {
 	return tokens[0].Peek(), nil
 }
 
-func (p *Provider) AuthClient() (*auth.Client, error) {
+func AuthClient() (*auth.Client, error) {
 	return auth.NewClient(
 		build.Issuer(),
 		build.ClientID(),
@@ -68,32 +60,34 @@ func (p *Provider) AuthClient() (*auth.Client, error) {
 	)
 }
 
-func (p *Provider) getAccessTokenFromAPIToken(
+func getAccessTokenFromAPIToken(
 	ctx context.Context,
 ) (*session.Token, error) {
-	apiTokenRaw := os.Getenv("DEVBOX_API_TOKEN")
-	if apiTokenRaw == "" {
-		return nil, nil
-	}
-
-	apiToken, err := typeid.Parse[ids.APIToken](apiTokenRaw)
-	if err != nil {
-		return nil, err
-	}
-
-	apiClient := api.NewClient(ctx, build.JetpackAPIHost(), &session.Token{})
-	response, err := apiClient.GetAccessToken(ctx, apiToken)
-	if err != nil {
-		return nil, err
-	}
-
-	// This is not the greatest. This token is missing id, refresh, etc.
-	// It may be better to change api.NewClient() to take a token string instead.
-	p.cachedAccessTokenFromAPIToken = &session.Token{
-		Token: oauth2.Token{
-			AccessToken: response.AccessToken,
-		},
+	if cachedAccessTokenFromAPIToken == nil {
+		apiTokenRaw := os.Getenv("DEVBOX_API_TOKEN")
+		if apiTokenRaw == "" {
+			return nil, nil
+		}
+
+		apiToken, err := typeid.Parse[ids.APIToken](apiTokenRaw)
+		if err != nil {
+			return nil, err
+		}
+
+		apiClient := api.NewClient(ctx, build.JetpackAPIHost(), &session.Token{})
+		response, err := apiClient.GetAccessToken(ctx, apiToken)
+		if err != nil {
+			return nil, err
+		}
+
+		// This is not the greatest. This token is missing id, refresh, etc.
+		// It may be better to change api.NewClient() to take a token string instead.
+		cachedAccessTokenFromAPIToken = &session.Token{
+			Token: oauth2.Token{
+				AccessToken: response.AccessToken,
+			},
+		}
 	}
 
-	return p.cachedAccessTokenFromAPIToken, nil
+	return cachedAccessTokenFromAPIToken, nil
 }