Skip to content

Commit d434bf5

Browse files
mikeland73gcurtis
andauthored
[nix-cache] Allow use of multiple caches. Split read vs write (#2059)
## Summary * Use new API * Allow devbox to use multiple caches for read. * Split read vs write caches ## How was it tested? Untested, needs new api to be deployed --------- Signed-off-by: Mike Landau <[email protected]> Co-authored-by: Greg Curtis <[email protected]>
1 parent 8b69475 commit d434bf5

File tree

8 files changed

+105
-54
lines changed

8 files changed

+105
-54
lines changed

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ require (
4141
github.com/zealic/go2node v0.1.0
4242
go.jetify.com/typeid v1.1.0
4343
go.jetpack.io/envsec v0.0.16-0.20240329013200-4174c0acdb00
44-
go.jetpack.io/pkg v0.0.0-20240425160511-7b1b3c860422
44+
go.jetpack.io/pkg v0.0.0-20240516160739-268b3dd5f65d
4545
golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
4646
golang.org/x/mod v0.16.0
4747
golang.org/x/oauth2 v0.19.0

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -364,6 +364,8 @@ go.jetpack.io/envsec v0.0.16-0.20240329013200-4174c0acdb00 h1:Kb+OlWOntAq+1nF+01
364364
go.jetpack.io/envsec v0.0.16-0.20240329013200-4174c0acdb00/go.mod h1:dVG2n8fBAGpQczW8yk/6wuXb9uEhzaJF7wGXkGLRRCU=
365365
go.jetpack.io/pkg v0.0.0-20240425160511-7b1b3c860422 h1:nycOky7HqHwm7yyRy8nM1vnehkPmpSQEjdVSp9oz0to=
366366
go.jetpack.io/pkg v0.0.0-20240425160511-7b1b3c860422/go.mod h1:+uFriYKQtj7KHG5g+n6EKspEBg4zlwvq7Z/YOmatjQU=
367+
go.jetpack.io/pkg v0.0.0-20240516160739-268b3dd5f65d h1:r9njTg75ZHdafCFs3kcXy3a3BHzIcvHA75s1HKXUTmQ=
368+
go.jetpack.io/pkg v0.0.0-20240516160739-268b3dd5f65d/go.mod h1:+uFriYKQtj7KHG5g+n6EKspEBg4zlwvq7Z/YOmatjQU=
367369
go.jetpack.io/typeid v1.0.1-0.20240410183543-96a4fd53d1e2 h1:w9uWg8BAim374iWzxEuDhf6MJ/cMQVR/0xi/L3DgfT0=
368370
go.jetpack.io/typeid v1.0.1-0.20240410183543-96a4fd53d1e2/go.mod h1:zqBUNjE3NxAL8wRYW2LcOpH5LxCvTqthm9YSuuu1ic4=
369371
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=

internal/boxcli/cache.go

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,16 @@ import (
77
"encoding/json"
88
"fmt"
99
"os/user"
10+
"slices"
1011

1112
"github.com/MakeNowJust/heredoc/v2"
1213
"github.com/pkg/errors"
14+
"github.com/samber/lo"
1315
"github.com/spf13/cobra"
1416
"go.jetpack.io/devbox/internal/devbox"
1517
"go.jetpack.io/devbox/internal/devbox/devopt"
1618
"go.jetpack.io/devbox/internal/devbox/providers/nixcache"
19+
nixv1alpha1 "go.jetpack.io/pkg/api/gen/priv/nix/v1alpha1"
1720
)
1821

1922
type cacheFlags struct {
@@ -118,16 +121,26 @@ func cacheInfoCmd() *cobra.Command {
118121
RunE: func(cmd *cobra.Command, args []string) error {
119122
// TODO(gcurtis): We can also output info about the daemon config status
120123
// here
121-
uri, err := nixcache.Get().URI(cmd.Context())
124+
caches, err := nixcache.Get().Caches(cmd.Context())
122125
if err != nil {
123126
return err
124127
}
125-
if uri == "" {
128+
if len(caches) == 0 {
126129
fmt.Fprintln(cmd.OutOrStdout(), "No cache configured")
127-
return nil
128130
}
129-
fmt.Fprintln(cmd.OutOrStdout(), "Cache URI:", uri)
130-
return err
131+
for _, cache := range caches {
132+
isReadOnly := !slices.Contains(
133+
cache.GetPermissions(),
134+
nixv1alpha1.Permission_PERMISSION_WRITE,
135+
)
136+
fmt.Fprintf(
137+
cmd.OutOrStdout(),
138+
"* %s %s\n",
139+
cache.GetUri(),
140+
lo.Ternary(isReadOnly, "(read-only)", ""),
141+
)
142+
}
143+
return nil
131144
},
132145
}
133146
}

internal/devbox/cache.go

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,14 @@ package devbox
22

33
import (
44
"context"
5+
"errors"
56
"io"
67

78
"go.jetpack.io/devbox/internal/boxcli/usererr"
89
"go.jetpack.io/devbox/internal/devbox/providers/nixcache"
910
"go.jetpack.io/devbox/internal/nix"
11+
"go.jetpack.io/devbox/internal/ux"
12+
"go.jetpack.io/pkg/auth"
1013
)
1114

1215
func (d *Devbox) UploadProjectToCache(
@@ -15,17 +18,14 @@ func (d *Devbox) UploadProjectToCache(
1518
) error {
1619
if cacheURI == "" {
1720
var err error
18-
cacheURI, err = d.providers.NixCache.URI(ctx)
21+
cacheURI, err = getWriteCacheURI(ctx, d.stderr, d.providers.NixCache)
1922
if err != nil {
2023
return err
2124
}
22-
if cacheURI == "" {
23-
return usererr.New("Your account's organization doesn't have a Nix cache.")
24-
}
2525
}
2626

2727
creds, err := d.providers.NixCache.Credentials(ctx)
28-
if err != nil {
28+
if err != nil && !errors.Is(err, auth.ErrNotLoggedIn) {
2929
return err
3030
}
3131
profilePath, err := d.profilePath()
@@ -50,18 +50,34 @@ func UploadInstallableToCache(
5050
) error {
5151
if cacheURI == "" {
5252
var err error
53-
cacheURI, err = nixcache.Get().URI(ctx)
53+
cacheURI, err = getWriteCacheURI(ctx, stderr, *nixcache.Get())
5454
if err != nil {
5555
return err
5656
}
57-
if cacheURI == "" {
58-
return usererr.New("Your account's organization doesn't have a Nix cache.")
59-
}
6057
}
6158

6259
creds, err := nixcache.Get().Credentials(ctx)
63-
if err != nil {
60+
if err != nil && !errors.Is(err, auth.ErrNotLoggedIn) {
6461
return err
6562
}
6663
return nix.CopyInstallableToCache(ctx, stderr, cacheURI, installable, creds.Env())
6764
}
65+
66+
func getWriteCacheURI(
67+
ctx context.Context,
68+
w io.Writer,
69+
provider nixcache.Provider,
70+
) (string, error) {
71+
caches, err := provider.WriteCaches(ctx)
72+
if err != nil {
73+
return "", err
74+
}
75+
if len(caches) == 0 {
76+
return "",
77+
usererr.New("You don't have permission to write to any Nix caches.")
78+
}
79+
if len(caches) > 1 {
80+
ux.Fwarning(w, "Multiple caches available, using %s.\n", caches[0].GetUri())
81+
}
82+
return caches[0].GetUri(), nil
83+
}

internal/devbox/packages.go

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
"go.jetpack.io/devbox/internal/redact"
2929
"go.jetpack.io/devbox/internal/shellgen"
3030
"go.jetpack.io/devbox/internal/telemetry"
31+
nixv1alpha1 "go.jetpack.io/pkg/api/gen/priv/nix/v1alpha1"
3132

3233
"go.jetpack.io/devbox/internal/boxcli/usererr"
3334
"go.jetpack.io/devbox/internal/debug"
@@ -444,14 +445,19 @@ func (d *Devbox) installNixPackagesToStore(ctx context.Context, mode installMode
444445
Flags: flags,
445446
Writer: d.stderr,
446447
}
447-
args.ExtraSubstituter, err = d.providers.NixCache.URI(ctx)
448+
caches, err := d.providers.NixCache.ReadCaches(ctx)
448449
if err != nil {
449450
debug.Log("error getting nix cache URI, assuming user doesn't have access: %v", err)
450451
}
451452

453+
args.ExtraSubstituters = lo.Map(
454+
caches, func(c *nixv1alpha1.NixBinCache, _ int) string {
455+
return c.GetUri()
456+
})
457+
452458
// TODO (Landau): handle errors that are not auth.ErrNotLoggedIn
453459
// Only lookup credentials if we have a cache to use
454-
if args.ExtraSubstituter != "" {
460+
if len(args.ExtraSubstituters) > 0 {
455461
creds, err := d.providers.NixCache.Credentials(ctx)
456462
if err == nil {
457463
args.Env = creds.Env()
@@ -469,7 +475,7 @@ func (d *Devbox) installNixPackagesToStore(ctx context.Context, mode installMode
469475
var daemonErr *nix.DaemonError
470476
if errors.As(err, &daemonErr) {
471477
// Error here to give the user a chance to restart the daemon.
472-
return usererr.New("Devbox configured Nix to use %q as a cache. Please restart the Nix daemon and re-run Devbox.", args.ExtraSubstituter)
478+
return usererr.New("Devbox configured Nix to use a new cache. Please restart the Nix daemon and re-run Devbox.")
473479
}
474480
// Other errors indicate we couldn't update nix.conf, so just warn and continue
475481
// by building from source if necessary.

internal/devbox/providers/nixcache/nixcache.go

Lines changed: 38 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@ package nixcache
22

33
import (
44
"context"
5+
"slices"
56
"time"
67

8+
"github.com/samber/lo"
79
"go.jetpack.io/devbox/internal/build"
810
"go.jetpack.io/devbox/internal/cachehash"
911
"go.jetpack.io/devbox/internal/devbox/providers/identity"
@@ -55,40 +57,49 @@ func (p *Provider) Credentials(ctx context.Context) (AWSCredentials, error) {
5557
return creds, nil
5658
}
5759

58-
// URI queries the Jetify API for the URI that points to user's private cache.
59-
// If their account doesn't have access to a cache, it returns an empty string
60-
// and a nil error.
61-
func (p *Provider) URI(ctx context.Context) (string, error) {
62-
cache := filecache.New[string]("devbox/providers/nixcache")
60+
func (p *Provider) Caches(
61+
ctx context.Context,
62+
) ([]*nixv1alpha1.NixBinCache, error) {
6363
token, err := identity.Get().GenSession(ctx)
6464
if err != nil {
65-
return "", err
65+
return nil, err
6666
}
67-
// Landau: I think we can probably remove this cache? This endpoint is very
68-
// fast and we only use this for build/upload which are slow.
69-
uri, err := cache.GetOrSet(
70-
"uri-"+getSubOrAccessTokenHash(token),
71-
func() (string, time.Duration, error) {
72-
client := api.NewClient(ctx, build.JetpackAPIHost(), token)
73-
resp, err := client.GetBinCache(ctx)
74-
if err != nil {
75-
return "", 0, redact.Errorf("nixcache: get uri: %w", redact.Safe(err))
76-
}
67+
client := api.NewClient(ctx, build.JetpackAPIHost(), token)
68+
resp, err := client.GetBinCache(ctx)
69+
if err != nil {
70+
return nil, redact.Errorf("nixcache: get caches: %w", redact.Safe(err))
71+
}
72+
return resp.GetCaches(), nil
73+
}
7774

78-
// Don't cache negative responses.
79-
if resp.GetNixBinCacheUri() == "" {
80-
return "", 0, nil
81-
}
75+
func (p *Provider) ReadCaches(
76+
ctx context.Context,
77+
) ([]*nixv1alpha1.NixBinCache, error) {
78+
caches, err := p.Caches(ctx)
79+
if err != nil {
80+
return nil, err
81+
}
82+
return lo.Filter(caches, func(c *nixv1alpha1.NixBinCache, _ int) bool {
83+
return slices.Contains(
84+
c.GetPermissions(),
85+
nixv1alpha1.Permission_PERMISSION_READ,
86+
)
87+
}), nil
88+
}
8289

83-
// TODO(gcurtis): do a better job of invalidating the URI after
84-
// a Nix command fails to query the cache.
85-
return resp.GetNixBinCacheUri(), 24 * time.Hour, nil
86-
},
87-
)
90+
func (p *Provider) WriteCaches(
91+
ctx context.Context,
92+
) ([]*nixv1alpha1.NixBinCache, error) {
93+
caches, err := p.Caches(ctx)
8894
if err != nil {
89-
return "", redact.Errorf("nixcache: get uri: %w", redact.Safe(err))
95+
return nil, err
9096
}
91-
return uri, nil
97+
return lo.Filter(caches, func(c *nixv1alpha1.NixBinCache, _ int) bool {
98+
return slices.Contains(
99+
c.GetPermissions(),
100+
nixv1alpha1.Permission_PERMISSION_WRITE,
101+
)
102+
}), nil
92103
}
93104

94105
// AWSCredentials are short-lived credentials that grant access to a private Nix

internal/devpkg/narinfo_cache.go

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,6 @@ func (p *Package) keyForOutput(output string) string {
128128
output = strings.Join(names, ",")
129129
}
130130
}
131-
fmt.Println("output: ", output)
132131

133132
return fmt.Sprintf("%s^%s", p.Raw, output)
134133
}

internal/nix/build.go

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,19 @@ import (
55
"io"
66
"os"
77
"os/exec"
8+
"strings"
89

910
"github.com/pkg/errors"
1011
"go.jetpack.io/devbox/internal/debug"
1112
"go.jetpack.io/devbox/internal/redact"
1213
)
1314

1415
type BuildArgs struct {
15-
AllowInsecure bool
16-
Env []string
17-
ExtraSubstituter string
18-
Flags []string
19-
Writer io.Writer
16+
AllowInsecure bool
17+
Env []string
18+
ExtraSubstituters []string
19+
Flags []string
20+
Writer io.Writer
2021
}
2122

2223
func Build(ctx context.Context, args *BuildArgs, installables ...string) error {
@@ -26,8 +27,11 @@ func Build(ctx context.Context, args *BuildArgs, installables ...string) error {
2627
cmd.Args = append(cmd.Args, installables...)
2728
// Adding extra substituters only here to be conservative, but this could also
2829
// be added to ExperimentalFlags() in the future.
29-
if args.ExtraSubstituter != "" {
30-
cmd.Args = append(cmd.Args, "--extra-substituters", args.ExtraSubstituter)
30+
if len(args.ExtraSubstituters) > 0 {
31+
cmd.Args = append(cmd.Args,
32+
"--extra-substituters",
33+
strings.Join(args.ExtraSubstituters, " "),
34+
)
3135
}
3236
cmd.Env = append(allowUnfreeEnv(os.Environ()), args.Env...)
3337
if args.AllowInsecure {

0 commit comments

Comments
 (0)