diff --git a/devbox.lock b/devbox.lock index a5a1dbd8900..3e972c85304 100644 --- a/devbox.lock +++ b/devbox.lock @@ -2,8 +2,8 @@ "lockfile_version": "1", "packages": { "fd@latest": { - "last_modified": "2025-02-07T11:26:36Z", - "resolved": "github:NixOS/nixpkgs/d98abf5cf5914e5e4e9d57205e3af55ca90ffc1d#fd", + "last_modified": "2025-03-11T17:52:14Z", + "resolved": "github:NixOS/nixpkgs/0d534853a55b5d02a4ababa1d71921ce8f0aee4c#fd", "source": "devbox-search", "version": "10.2.0", "systems": { @@ -11,164 +11,165 @@ "outputs": [ { "name": "out", - "path": "/nix/store/ad5m54pfn9k39v80lpyhrnsh336nqrp5-fd-10.2.0", + "path": "/nix/store/40pdazk980kp3h26py4hjyx9rys1g14n-fd-10.2.0", "default": true } ], - "store_path": "/nix/store/ad5m54pfn9k39v80lpyhrnsh336nqrp5-fd-10.2.0" + "store_path": "/nix/store/40pdazk980kp3h26py4hjyx9rys1g14n-fd-10.2.0" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/vn3mny38qmf3lm809rpcvahxqwhkqb7m-fd-10.2.0", + "path": "/nix/store/76zcwa1d33vciy4gyqvk6jl2n3g1542q-fd-10.2.0", "default": true } ], - "store_path": "/nix/store/vn3mny38qmf3lm809rpcvahxqwhkqb7m-fd-10.2.0" + "store_path": "/nix/store/76zcwa1d33vciy4gyqvk6jl2n3g1542q-fd-10.2.0" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/6kdv4iy9j3svncq0vs47wxfvvn7flcr5-fd-10.2.0", + "path": "/nix/store/ys9qmljs0ag7j040radgg48l6pvjmv9l-fd-10.2.0", "default": true } ], - "store_path": "/nix/store/6kdv4iy9j3svncq0vs47wxfvvn7flcr5-fd-10.2.0" + "store_path": "/nix/store/ys9qmljs0ag7j040radgg48l6pvjmv9l-fd-10.2.0" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/x58pg72qw2xv1vvs4pbqw63zhkdkp331-fd-10.2.0", + "path": "/nix/store/rrdvpl7rym4ia0h7rfz1vmlcvvivj30j-fd-10.2.0", "default": true } ], - "store_path": "/nix/store/x58pg72qw2xv1vvs4pbqw63zhkdkp331-fd-10.2.0" + "store_path": "/nix/store/rrdvpl7rym4ia0h7rfz1vmlcvvivj30j-fd-10.2.0" } } }, "git@latest": { - "last_modified": "2025-02-07T11:26:36Z", - "resolved": "github:NixOS/nixpkgs/d98abf5cf5914e5e4e9d57205e3af55ca90ffc1d#git", + "last_modified": "2025-03-11T17:52:14Z", + "resolved": "github:NixOS/nixpkgs/0d534853a55b5d02a4ababa1d71921ce8f0aee4c#git", "source": "devbox-search", - "version": "2.47.2", + "version": "2.48.1", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/9z3jhc0rlj3zaw8nd1zka9vli6w0q11g-git-2.47.2", + "path": "/nix/store/b3sci30zzzlj3rzj1y89cijnd6zcwapk-git-2.48.1", "default": true }, { "name": "doc", - "path": "/nix/store/rh151iwgy4h8yv8kxd5facw57cyj0bav-git-2.47.2-doc" + "path": "/nix/store/086knqdw7fjgzczp0i6nad95s2v6jbya-git-2.48.1-doc" } ], - "store_path": "/nix/store/9z3jhc0rlj3zaw8nd1zka9vli6w0q11g-git-2.47.2" + "store_path": "/nix/store/b3sci30zzzlj3rzj1y89cijnd6zcwapk-git-2.48.1" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/gx5y37qcfqdvn0h6swjd04dmqjjh3nk7-git-2.47.2", + "path": "/nix/store/pck1dr5jxrd5b8nmfasbn13z422jhcfm-git-2.48.1", "default": true }, { "name": "debug", - "path": "/nix/store/8vfpmf3vjgzl2psip76p0f9h11sb6y3p-git-2.47.2-debug" + "path": "/nix/store/xqqsvzlilh843rm6knykyng81apapr33-git-2.48.1-debug" }, { "name": "doc", - "path": "/nix/store/c25mq3q83dvw3k5pb0qr5333g3cycylq-git-2.47.2-doc" + "path": "/nix/store/485b32ys0s2dvjfisn7405ildmpqvfzk-git-2.48.1-doc" } ], - "store_path": "/nix/store/gx5y37qcfqdvn0h6swjd04dmqjjh3nk7-git-2.47.2" + "store_path": "/nix/store/pck1dr5jxrd5b8nmfasbn13z422jhcfm-git-2.48.1" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/39xx5gx3hxigs1b5ldw5i2jr84vsn3rf-git-2.47.2", + "path": "/nix/store/9qjzgsf9mvdp6sfd7xyzhgrahl2qhhp6-git-2.48.1", "default": true }, { "name": "doc", - "path": "/nix/store/xmh2djjrnbpiqqgpblrcbavnqh0nv4km-git-2.47.2-doc" + "path": "/nix/store/cgv7qa0ix059ma9a0qac0bywfvl3k7k2-git-2.48.1-doc" } ], - "store_path": "/nix/store/39xx5gx3hxigs1b5ldw5i2jr84vsn3rf-git-2.47.2" + "store_path": "/nix/store/9qjzgsf9mvdp6sfd7xyzhgrahl2qhhp6-git-2.48.1" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/33g65w5cc9n8fr0hxj84282xmv4l7hyl-git-2.47.2", + "path": "/nix/store/lqx2rv26sdndpa2vyy2vxsahj03km69z-git-2.48.1", "default": true }, { - "name": "debug", - "path": "/nix/store/jyz4nvcd3bci4vg2sfsmvrq0fp9mzr5a-git-2.47.2-debug" + "name": "doc", + "path": "/nix/store/hjczhs1dm3hzij7mx5c91rkzqvkb89av-git-2.48.1-doc" }, { - "name": "doc", - "path": "/nix/store/lb4nipdhlwrxdavz7gdkcik6lkz3cbdm-git-2.47.2-doc" + "name": "debug", + "path": "/nix/store/bk8xndavdnc2qgyvc6hcc8h29lk9jzqb-git-2.48.1-debug" } ], - "store_path": "/nix/store/33g65w5cc9n8fr0hxj84282xmv4l7hyl-git-2.47.2" + "store_path": "/nix/store/lqx2rv26sdndpa2vyy2vxsahj03km69z-git-2.48.1" } } }, "github:NixOS/nixpkgs/nixpkgs-unstable": { - "resolved": "github:NixOS/nixpkgs/73cf49b8ad837ade2de76f87eb53fc85ed5d4680?lastModified=1739866667&narHash=sha256-EO1ygNKZlsAC9avfcwHkKGMsmipUk1Uc0TbrEZpkn64%3D" + "last_modified": "2025-04-07T13:23:10Z", + "resolved": "github:NixOS/nixpkgs/b0b4b5f8f621bfe213b8b21694bab52ecfcbf30b?lastModified=1744032190&narHash=sha256-KSlfrncSkcu1YE%2BuuJ%2FPTURsSlThoGkRqiGDVdbiE%2Fk%3D" }, "go@latest": { - "last_modified": "2025-02-12T00:10:52Z", - "resolved": "github:NixOS/nixpkgs/83a2581c81ff5b06f7c1a4e7cc736a455dfcf7b4#go_1_24", + "last_modified": "2025-03-11T17:52:14Z", + "resolved": "github:NixOS/nixpkgs/0d534853a55b5d02a4ababa1d71921ce8f0aee4c#go", "source": "devbox-search", - "version": "1.24.0", + "version": "1.24.1", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/qldcnifalkvyah0wnv7m4zb854yd9l88-go-1.24.0", + "path": "/nix/store/ja4jxx60lh1qfqfl4z4p2rff56ia1c3c-go-1.24.1", "default": true } ], - "store_path": "/nix/store/qldcnifalkvyah0wnv7m4zb854yd9l88-go-1.24.0" + "store_path": "/nix/store/ja4jxx60lh1qfqfl4z4p2rff56ia1c3c-go-1.24.1" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/rrxgml7w4pfmibjbspkdvrw8vd2vnarb-go-1.24.0", + "path": "/nix/store/8ply43gnxk1xwichr81mpgbjcd9a1y5w-go-1.24.1", "default": true } ], - "store_path": "/nix/store/rrxgml7w4pfmibjbspkdvrw8vd2vnarb-go-1.24.0" + "store_path": "/nix/store/8ply43gnxk1xwichr81mpgbjcd9a1y5w-go-1.24.1" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/7imv22pl4qrjwvi6jzlfb305rc2min45-go-1.24.0", + "path": "/nix/store/87yxrfx5lh78bdz393i33cr5z23x06q4-go-1.24.1", "default": true } ], - "store_path": "/nix/store/7imv22pl4qrjwvi6jzlfb305rc2min45-go-1.24.0" + "store_path": "/nix/store/87yxrfx5lh78bdz393i33cr5z23x06q4-go-1.24.1" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/vh5d5bj1sljdhdypy80x1ydx2jx6rv2q-go-1.24.0", + "path": "/nix/store/cfjhl0kn7xc65466pha9fkrvigw3g72n-go-1.24.1", "default": true } ], - "store_path": "/nix/store/vh5d5bj1sljdhdypy80x1ydx2jx6rv2q-go-1.24.0" + "store_path": "/nix/store/cfjhl0kn7xc65466pha9fkrvigw3g72n-go-1.24.1" } } } diff --git a/internal/devbox/update.go b/internal/devbox/update.go index 79a30d9774d..91eba0125b9 100644 --- a/internal/devbox/update.go +++ b/internal/devbox/update.go @@ -6,6 +6,7 @@ package devbox import ( "context" "fmt" + "slices" "github.com/pkg/errors" "go.jetify.com/devbox/internal/devbox/devopt" @@ -20,6 +21,20 @@ import ( ) func (d *Devbox) Update(ctx context.Context, opts devopt.UpdateOpts) error { + if len(opts.Pkgs) == 0 || slices.Contains(opts.Pkgs, "nixpkgs") { + if err := d.lockfile.UpdateStdenv(); err != nil { + return err + } + // if nixpkgs is the only package to update, just return here. + if len(opts.Pkgs) == 1 { + return nil + } + // Otherwise, remove nixpkgs and continue + opts.Pkgs = slices.DeleteFunc(opts.Pkgs, func(pkg string) bool { + return pkg == "nixpkgs" + }) + } + inputs, err := d.inputsToUpdate(opts) if err != nil { return err @@ -65,9 +80,6 @@ func (d *Devbox) Update(ctx context.Context, opts devopt.UpdateOpts) error { } } - if err := d.updateStdenv(); err != nil { - return err - } mode := update if opts.NoInstall { mode = noInstall @@ -110,15 +122,6 @@ func (d *Devbox) inputsToUpdate( return pkgsToUpdate, nil } -func (d *Devbox) updateStdenv() error { - err := d.lockfile.Remove(d.Stdenv().String()) - if err != nil { - return err - } - d.lockfile.Stdenv() // will re-resolve the stdenv flake - return nil -} - func (d *Devbox) updateDevboxPackage(pkg *devpkg.Package) error { resolved, err := d.lockfile.FetchResolvedPackage(pkg.Raw) if err != nil { diff --git a/internal/lock/lockfile.go b/internal/lock/lockfile.go index 3fb0b8ff605..a3fd3070bbe 100644 --- a/internal/lock/lockfile.go +++ b/internal/lock/lockfile.go @@ -141,6 +141,23 @@ func (f *File) Save() error { return cuecfg.WriteFile(lockFilePath(f.devboxProject.ProjectDir()), f) } +func (f *File) UpdateStdenv() error { + if err := nix.ClearFlakeCache(f.devboxProject.Stdenv()); err != nil { + return err + } + if err := f.Remove(f.devboxProject.Stdenv().String()); err != nil { + return err + } + return f.Add(f.devboxProject.Stdenv().String()) +} + +// TODO: We should improve a few issues with this function: +// * It shared the same name as Devbox.Stdenv() which is confusing. +// * Since File implements DevboxProject, IDEs really struggle to accurately find call sites. +// (side note, we should remove DevboxProject interface) +// * This function forces a resolution of the stdenv flake which is slow and doesn't give us a +// chance to "prep" the user for some waiting. +// * Should we rename to Nixpkgs() ? Stdenv feels a bit ambiguous. func (f *File) Stdenv() flake.Ref { unlocked := f.devboxProject.Stdenv() pkg, err := f.Resolve(unlocked.String()) diff --git a/internal/lock/resolve.go b/internal/lock/resolve.go index 91c2cef3fa7..38873d5d127 100644 --- a/internal/lock/resolve.go +++ b/internal/lock/resolve.go @@ -39,7 +39,8 @@ func (f *File) FetchResolvedPackage(pkg string) (*Package, error) { return nil, err } return &Package{ - Resolved: installable.String(), + Resolved: installable.String(), + LastModified: time.Unix(installable.Ref.LastModified, 0).UTC().Format(time.RFC3339), }, nil } @@ -220,7 +221,25 @@ func lockFlake(ctx context.Context, ref flake.Ref) (flake.Ref, error) { return ref, nil } - meta, err := nix.ResolveFlake(ctx, ref) + var meta nix.FlakeMetadata + var err error + // For nixpkgs, we cache resolutions (currently flakeCacheTTL=30 days) to avoid downloading + // new nixpkgs too often which is really slow and rarely changes anything. + // + // Ideally we can do something similar for all packages (flake and otherwise) + // Specifically, if user adds python@3.12 (or python@latest that resolves to 3.12) and that + // package is already installed, we should use it instead of using 3.12 from search service + // (which may have different store path). This would allow all devbox projects to share packages + // if the version resolution is the same. + // + // That said, the logic for caching resolved versions and non-locked flake references would not + // be the same. + if ref.IsNixpkgs() { + meta, err = nix.ResolveCachedFlake(ctx, ref) + } else { + meta, err = nix.ResolveFlake(ctx, ref) + } + if err != nil { return ref, err } diff --git a/internal/nix/flake.go b/internal/nix/flake.go index 8b196a9ddae..822a91a5c3b 100644 --- a/internal/nix/flake.go +++ b/internal/nix/flake.go @@ -3,16 +3,23 @@ package nix import ( "context" "encoding/json" + "time" "go.jetify.com/devbox/nix/flake" + "go.jetify.com/pkg/filecache" ) +const flakeCacheTTL = time.Hour * 24 * 30 + +var flakeFileCache = filecache.New[FlakeMetadata]("devbox/flakes") + type FlakeMetadata struct { - Description string `json:"description"` - Original flake.Ref `json:"original"` - Resolved flake.Ref `json:"resolved"` - Locked flake.Ref `json:"locked"` - Path string `json:"path"` + Description string `json:"description"` + LastModified int64 `json:"lastModified"` + Locked flake.Ref `json:"locked"` + Original flake.Ref `json:"original"` + Path string `json:"path"` + Resolved flake.Ref `json:"resolved"` } func ResolveFlake(ctx context.Context, ref flake.Ref) (FlakeMetadata, error) { @@ -28,3 +35,18 @@ func ResolveFlake(ctx context.Context, ref flake.Ref) (FlakeMetadata, error) { } return meta, nil } + +func ResolveCachedFlake(ctx context.Context, ref flake.Ref) (FlakeMetadata, error) { + return flakeFileCache.GetOrSet(ref.String(), func() (FlakeMetadata, time.Duration, error) { + meta, err := ResolveFlake(ctx, ref) + if err != nil { + return FlakeMetadata{}, 0, err + } + return meta, flakeCacheTTL, nil + }) +} + +func ClearFlakeCache(ref flake.Ref) error { + // TODO: Add unset to filecache + return flakeFileCache.Set(ref.String(), FlakeMetadata{}, -1) +} diff --git a/internal/shellgen/flake_input.go b/internal/shellgen/flake_input.go index f4fe433f6be..03a9b97e913 100644 --- a/internal/shellgen/flake_input.go +++ b/internal/shellgen/flake_input.go @@ -22,28 +22,12 @@ type flakeInput struct { Ref flake.Ref } -// IsNixpkgs reports whether the input looks like a nixpkgs flake. -// -// While there are many ways to specify this input, devbox always uses -// github:NixOS/nixpkgs/ as the URL. If the user wishes to reference nixpkgs -// themselves, this function may not return true. -func (f *flakeInput) IsNixpkgs() bool { - switch f.Ref.Type { - case flake.TypeGitHub: - return f.Ref.Owner == "NixOS" && f.Ref.Repo == "nixpkgs" - case flake.TypeIndirect: - return f.Ref.ID == "nixpkgs" - default: - return false - } -} - func (f *flakeInput) HashFromNixPkgsURL() string { return f.Ref.Rev } func (f *flakeInput) URLWithCaching() string { - if !f.IsNixpkgs() { + if !f.Ref.IsNixpkgs() { return nix.FixInstallableArg(f.Ref.String()) } return getNixpkgsInfo(f.Ref.Rev).URL @@ -99,7 +83,7 @@ func (f *flakeInput) BuildInputsForSymlinkJoin() ([]*SymlinkJoin, error) { joins = append(joins, &SymlinkJoin{ Name: pkg.String() + "-combined", Paths: lo.Map(outputNames, func(outputName string, _ int) string { - if !f.IsNixpkgs() { + if !f.Ref.IsNixpkgs() { return f.Name + "." + attributePath + "." + outputName } parts := strings.Split(attributePath, ".") @@ -140,7 +124,7 @@ func (f *flakeInput) BuildInputs() ([]string, error) { if err != nil { return nil, err } - if !f.IsNixpkgs() { + if !f.Ref.IsNixpkgs() { return lo.Map(attributePaths, func(pkg string, _ int) string { return f.Name + "." + pkg }), nil diff --git a/internal/shellgen/tmpl/flake.nix.tmpl b/internal/shellgen/tmpl/flake.nix.tmpl index d0ede605761..3bd0513ff4d 100644 --- a/internal/shellgen/tmpl/flake.nix.tmpl +++ b/internal/shellgen/tmpl/flake.nix.tmpl @@ -19,7 +19,7 @@ let pkgs = nixpkgs.legacyPackages.{{ .System }}; {{- range $_, $flake := .FlakeInputs }} - {{- if .IsNixpkgs }} + {{- if $flake.Ref.IsNixpkgs }} {{.PkgImportName}} = (import {{.Name}} { system = "{{ $.System }}"; config.allowUnfree = true; diff --git a/nix/flake/flakeref.go b/nix/flake/flakeref.go index 0dab094579f..6e9f545959b 100644 --- a/nix/flake/flakeref.go +++ b/nix/flake/flakeref.go @@ -465,6 +465,22 @@ func (r Ref) String() string { } } +// IsNixpkgs reports whether the flake reference looks like a nixpkgs flake. +// +// While there are many ways to specify this input, devbox always uses +// github:NixOS/nixpkgs/ as the URL. If the user wishes to reference nixpkgs +// themselves, this function may not return true. +func (r Ref) IsNixpkgs() bool { + switch r.Type { + case TypeGitHub: + return r.Owner == "NixOS" && r.Repo == "nixpkgs" + case TypeIndirect: + return r.ID == "nixpkgs" + default: + return false + } +} + func isGitHash(s string) bool { if len(s) != 40 { return false