[cache] Fallback for failed build with cache (#2082)

## Summary

* If we fail to build with caches, try again without them.
* Added logging when build fails and user has substituters.

## How was it tested?

Ran install on devbox project that installs cached terraform. Commented
out `--extra-substituters` flag to force it to fail. Observed output
indicating substituter error, and fallback to building from source.

<img width="935" alt="image"
src="https://github.com/jetify-com/devbox/assets/544948/0c1e6863-d506-4c11-b2d3-8556313be587">

Author: mikeland73

internal/devbox/packages.go

@@ -16,7 +16,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/fatih/color"
 	"github.com/pkg/errors"
 	"github.com/samber/lo"
 	"go.jetpack.io/devbox/internal/devbox/devopt"
@@ -404,12 +403,26 @@ func (d *Devbox) installPackages(ctx context.Context, mode installMode) error {
 	}
 
 	if err := d.installNixPackagesToStore(ctx, mode); err != nil {
+		if caches, _ := nixcache.CachedReadCaches(ctx); len(caches) > 0 {
+			err = d.handleInstallFailure(ctx, mode)
+		}
 		return err
 	}
 
 	return d.InstallRunXPackages(ctx)
 }
 
+func (d *Devbox) handleInstallFailure(ctx context.Context, mode installMode) error {
+	ux.Fwarning(d.stderr, "Failed to build from cache, building from source.\n")
+	telemetry.Event(telemetry.EventNixBuildWithSubstitutersFailed, telemetry.Metadata{
+		Packages: lo.Map(
+			d.InstallablePackages(), func(p *devpkg.Package, _ int) string { return p.Raw }),
+	})
+	nixcache.DisableReadCaches()
+	devpkg.ClearNarInfoCache()
+	return d.installNixPackagesToStore(ctx, mode)
+}
+
 func (d *Devbox) InstallRunXPackages(ctx context.Context) error {
 	for _, pkg := range lo.Filter(d.InstallablePackages(), devpkg.IsRunX) {
 		lockedPkg, err := d.lockfile.Resolve(pkg.Raw)
@@ -516,7 +529,6 @@ func (d *Devbox) installNixPackagesToStore(ctx context.Context, mode installMode
 		args.AllowInsecure = allowInsecure
 		err = nix.Build(ctx, args, installables...)
 		if err != nil {
-			color.New(color.FgRed).Fprintf(d.stderr, "Fail\n")
 			return err
 		}
 		telemetry.Event(telemetry.EventNixBuildSuccess, telemetry.Metadata{

internal/devbox/providers/nixcache/nixcache.go

@@ -101,6 +101,14 @@ func CachedReadCaches(ctx context.Context) ([]*nixv1alpha1.NixBinCache, error) {
 	return cachedReadCaches.Do(ctx)
 }
 
+func DisableReadCaches() {
+	cachedReadCaches = goutil.OnceWithContext(
+		func(ctx context.Context) ([]*nixv1alpha1.NixBinCache, error) {
+			return nil, nil
+		},
+	)
+}
+
 func WriteCaches(
 	ctx context.Context,
 ) ([]*nixv1alpha1.NixBinCache, error) {

internal/devpkg/narinfo_cache.go

@@ -323,3 +323,7 @@ func readCaches(ctx context.Context) ([]string, error) {
 	}
 	return cacheURIs, nil
 }
+
+func ClearNarInfoCache() {
+	narInfoStatusFnCache = sync.Map{}
+}

internal/telemetry/telemetry.go

@@ -42,6 +42,7 @@ const (
 	EventShellInteractive
 	EventShellReady
 	EventNixBuildSuccess
+	EventNixBuildWithSubstitutersFailed
 )
 
 var (