Skip to content

flake: add narHash and lastModified attributes #2464

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Dec 20, 2024
Merged

flake: add narHash and lastModified attributes #2464

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ff89ae2
flake: add narHash and lastModified attributes
gcurtis Dec 20, 2024
b321c8e
buildQueryString -> appendQueryString
gcurtis Dec 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
191 changes: 162 additions & 29 deletions nix/flake/flakeref.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@
package flake

import (
"cmp"
"net/url"
"path"
"slices"
"strconv"
"strings"

"go.jetpack.io/devbox/internal/redact"
Expand Down Expand Up @@ -67,6 +69,14 @@ type Ref struct {
// or "git". Note that the URL is not the same as the raw unparsed
// flake ref.
URL string `json:"url,omitempty"`

// NARHash is the SRI hash of the flake's source. Specify a NAR hash to
// lock flakes that don't otherwise have a revision (such as "path" or
// "tarball" flakes).
NARHash string `json:"narHash,omitempty"`

// LastModified is the last modification time of the flake.
LastModified int64 `json:"lastModified,omitempty"`
}

// ParseRef parses a raw flake reference. Nix supports a variety of flake ref
Expand Down Expand Up @@ -156,38 +166,79 @@ func parseURLRef(ref string) (parsed Ref, fragment string, err error) {
} else {
parsed.Path = refURL.Path
}

query := refURL.Query()
parsed.NARHash = query.Get("narHash")
parsed.LastModified, err = atoiOmitZero(query.Get("lastModified"))
if err != nil {
return Ref{}, "", redact.Errorf("parse flake reference URL query parameter: lastModified=%s: %v", redact.Safe(parsed.LastModified), redact.Safe(err))
}
case "http", "https", "file":
if isArchive(refURL.Path) {
parsed.Type = TypeTarball
} else {
parsed.Type = TypeFile
}
parsed.Dir = refURL.Query().Get("dir")
query := refURL.Query()
parsed.Dir = query.Get("dir")
parsed.NARHash = query.Get("narHash")
parsed.LastModified, err = atoiOmitZero(query.Get("lastModified"))
if err != nil {
return Ref{}, "", redact.Errorf("parse flake reference URL query parameter: lastModified=%s: %v", redact.Safe(parsed.LastModified), redact.Safe(err))
}

// lastModified and narHash get stripped from the query
// parameters, but dir stays.
query.Del("lastModified")
query.Del("narHash")
refURL.RawQuery = query.Encode()
parsed.URL = refURL.String()
case "tarball+http", "tarball+https", "tarball+file":
parsed.Type = TypeTarball
parsed.Dir = refURL.Query().Get("dir")
query := refURL.Query()
parsed.Dir = query.Get("dir")
parsed.NARHash = query.Get("narHash")
parsed.LastModified, err = atoiOmitZero(query.Get("lastModified"))
if err != nil {
return Ref{}, "", redact.Errorf("parse flake reference URL query parameter: lastModified=%s: %v", redact.Safe(parsed.LastModified), redact.Safe(err))
}

// lastModified and narHash get stripped from the query
// parameters, but dir stays.
query.Del("lastModified")
query.Del("narHash")
refURL.RawQuery = query.Encode()
refURL.Scheme = refURL.Scheme[8:] // remove tarball+
parsed.URL = refURL.String()
case "file+http", "file+https", "file+file":
parsed.Type = TypeFile
parsed.Dir = refURL.Query().Get("dir")
query := refURL.Query()
parsed.Dir = query.Get("dir")
parsed.NARHash = query.Get("narHash")
parsed.LastModified, err = atoiOmitZero(query.Get("lastModified"))
if err != nil {
return Ref{}, "", redact.Errorf("parse flake reference URL query parameter: lastModified=%s: %v", redact.Safe(parsed.LastModified), redact.Safe(err))
}

// lastModified and narHash get stripped from the query
// parameters, but dir stays.
query.Del("lastModified")
query.Del("narHash")
refURL.RawQuery = query.Encode()
refURL.Scheme = refURL.Scheme[5:] // remove file+
parsed.URL = refURL.String()
case "git", "git+http", "git+https", "git+ssh", "git+git", "git+file":
parsed.Type = TypeGit
q := refURL.Query()
parsed.Dir = q.Get("dir")
parsed.Ref = q.Get("ref")
parsed.Rev = q.Get("rev")
query := refURL.Query()
parsed.Dir = query.Get("dir")
parsed.Ref = query.Get("ref")
parsed.Rev = query.Get("rev")

// ref and rev get stripped from the query parameters, but dir
// stays.
q.Del("ref")
q.Del("rev")
refURL.RawQuery = q.Encode()
query.Del("ref")
query.Del("rev")
refURL.RawQuery = query.Encode()
if len(refURL.Scheme) > 3 {
refURL.Scheme = refURL.Scheme[4:] // remove git+
}
Expand Down Expand Up @@ -245,9 +296,42 @@ func parseGitHubRef(refURL *url.URL, parsed *Ref) error {
parsed.Rev = qRev
}
parsed.Dir = refURL.Query().Get("dir")
parsed.NARHash = refURL.Query().Get("narHash")
return nil
}

// Locked reports whether r is locked. Locked flake references always resolve to
// the same content. For some flake types, determining if a Ref is locked
// depends on the local Nix configuration. In these cases, Locked conservatively
// returns false.
func (r Ref) Locked() bool {
// Search for the implementations of InputScheme::isLocked in the nix
// source.
//
// https://github.com/search?q=repo%3ANixOS%2Fnix+language%3AC%2B%2B+symbol%3AisLocked&type=code

switch r.Type {
case TypeFile, TypePath, TypeTarball:
return r.NARHash != ""
case TypeGit:
return r.Rev != ""
case TypeGitHub:
// We technically can't determine if a github flake is locked
// unless we know the trust-tarballs-from-git-forges Nix setting
// (which defaults to true), so we have to be conservative and
// check for rev and narHash.
//
// https://github.com/NixOS/nix/blob/3f3feae33e3381a2ea5928febe03329f0a578b20/src/libfetchers/github.cc#L304-L313
return r.Rev != "" && r.NARHash != ""
case TypeIndirect:
// Never locked because they must be resolved against a flake
// registry.
return false
default:
return false
}
}

// String encodes the flake reference as a URL-like string. It normalizes the
// result such that if two Ref values are equal, then their strings will also be
// equal.
Expand All @@ -262,15 +346,26 @@ func parseGitHubRef(refURL *url.URL, parsed *Ref) error {
// put in the path.
// - query parameters are sorted by key.
//
// If f is missing a type or has any invalid fields, String returns an empty
// If r is missing a type or has any invalid fields, String returns an empty
// string.
func (r Ref) String() string {
switch r.Type {
case TypeFile:
if r.URL == "" {
return ""
}
return "file+" + r.URL

url, err := url.Parse("file+" + r.URL)
if err != nil {
// This should be rare and only happen if the caller
// messed with the parsed URL.
return ""
}
url.RawQuery = appendQueryString(url.Query(),
"lastModified", itoaOmitZero(r.LastModified),
"narHash", r.NARHash,
Comment on lines +365 to +366
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what happens if lastModified or NarHash are zero?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

buildQueryString will omit them.

)
return url.String()
case TypeGit:
if r.URL == "" {
return ""
Expand All @@ -292,26 +387,35 @@ func (r Ref) String() string {
// messed with the parsed URL.
return ""
}
url.RawQuery = buildQueryString("ref", r.Ref, "rev", r.Rev, "dir", r.Dir)
url.RawQuery = appendQueryString(url.Query(), "ref", r.Ref, "rev", r.Rev, "dir", r.Dir)
return url.String()
case TypeGitHub:
if r.Owner == "" || r.Repo == "" {
return ""
}
url := &url.URL{
Scheme: "github",
Opaque: buildEscapedPath(r.Owner, r.Repo, r.Rev, r.Ref),
RawQuery: buildQueryString("host", r.Host, "dir", r.Dir),
Scheme: "github",
Opaque: buildEscapedPath(r.Owner, r.Repo, r.Rev, r.Ref),
RawQuery: appendQueryString(nil,
"host", r.Host,
"dir", r.Dir,
"lastModified", itoaOmitZero(r.LastModified),
"narHash", r.NARHash,
),
}
return url.String()
case TypeIndirect:
if r.ID == "" {
return ""
}
url := &url.URL{
Scheme: "flake",
Opaque: buildEscapedPath(r.ID, r.Ref, r.Rev),
RawQuery: buildQueryString("dir", r.Dir),
Scheme: "flake",
Opaque: buildEscapedPath(r.ID, r.Ref, r.Rev),
RawQuery: appendQueryString(nil,
"dir", r.Dir,
"lastModified", itoaOmitZero(r.LastModified),
"narHash", r.NARHash,
),
}
return url.String()
case TypePath:
Expand All @@ -330,6 +434,11 @@ func (r Ref) String() string {
} else if r.Path == "." {
url.Opaque = "."
}

url.RawQuery = appendQueryString(nil,
"lastModified", itoaOmitZero(r.LastModified),
"narHash", r.NARHash,
)
return url.String()
case TypeTarball:
if r.URL == "" {
Expand All @@ -338,17 +447,18 @@ func (r Ref) String() string {
if !strings.HasPrefix(r.URL, "tarball") {
r.URL = "tarball+" + r.URL
}
if r.Dir == "" {
return r.URL
}

url, err := url.Parse(r.URL)
if err != nil {
// This should be rare and only happen if the caller
// messed with the parsed URL.
return ""
}
url.RawQuery = buildQueryString("dir", r.Dir)
url.RawQuery = appendQueryString(url.Query(),
"dir", r.Dir,
"lastModified", itoaOmitZero(r.LastModified),
"narHash", r.NARHash,
)
return url.String()
default:
return ""
Expand Down Expand Up @@ -423,21 +533,44 @@ func buildEscapedPath(elem ...string) string {
return u.JoinPath(elem...).String()
}

// buildQueryString builds a URL query string from a list of key-value string
// appendQueryString builds a URL query string from a list of key-value string
// pairs, omitting any keys with empty values.
func buildQueryString(keyval ...string) string {
func appendQueryString(query url.Values, keyval ...string) string {
if len(keyval)%2 != 0 {
panic("buildQueryString: odd number of key-value pairs")
panic("appendQueryString: odd number of key-value pairs")
}

query := make(url.Values, len(keyval)/2)
appended := make(url.Values, len(query)+len(keyval)/2)
for k, vals := range query {
v := cmp.Or(vals...)
if v != "" {
appended.Set(k, v)
}
}
for i := 0; i < len(keyval); i += 2 {
k, v := keyval[i], keyval[i+1]
if v != "" {
query.Set(k, v)
appended.Set(k, v)
}
}
return query.Encode()
return appended.Encode()
}

// itoaOmitZero returns an empty string if i == 0, otherwise it formats i as a
// string in base-10.
func itoaOmitZero(i int64) string {
if i == 0 {
return ""
}
return strconv.FormatInt(i, 10)
}

// atoiOmitZero returns 0 if s == "", otherwised it parses s as a base-10 int64.
func atoiOmitZero(s string) (int64, error) {
if s == "" {
return 0, nil
}
return strconv.ParseInt(s, 10, 64)
}

// Special values for [Installable].Outputs.
Expand Down
24 changes: 14 additions & 10 deletions nix/flake/flakeref_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ func TestParseFlakeRef(t *testing.T) {
"path:/": {Type: TypePath, Path: "/"},
"path:/flake": {Type: TypePath, Path: "/flake"},
"path:/absolute/flake": {Type: TypePath, Path: "/absolute/flake"},
"path:/absolute/flake?lastModified=1734435836&narHash=sha256-kMBQ5PRiFLagltK0sH%2B08aiNt3zGERC2297iB6vrvlU%3D": {Type: TypePath, Path: "/absolute/flake", LastModified: 1734435836, NARHash: "sha256-kMBQ5PRiFLagltK0sH+08aiNt3zGERC2297iB6vrvlU="},

// URL-like paths can omit the "./" prefix for relative
// directories.
Expand Down Expand Up @@ -65,6 +66,7 @@ func TestParseFlakeRef(t *testing.T) {
"github:NixOS/nix?rev=5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Rev: "5233fd2ba76a3accb5aaa999c00509a11fd0793c"},
"github:NixOS/nix?host=example.com": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Host: "example.com"},
"github:NixOS/nix?host=example.com&dir=subdir": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Host: "example.com", Dir: "subdir"},
"github:NixOS/nix?host=example.com&dir=subdir&lastModified=1734435836&narHash=sha256-kMBQ5PRiFLagltK0sH%2B08aiNt3zGERC2297iB6vrvlU%3D": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Host: "example.com", Dir: "subdir", NARHash: "sha256-kMBQ5PRiFLagltK0sH+08aiNt3zGERC2297iB6vrvlU="},

// The github type allows clone-style URLs. The username and
// host are ignored.
Expand All @@ -81,7 +83,8 @@ func TestParseFlakeRef(t *testing.T) {
"git+ssh://[email protected]/repo/flake": {Type: TypeGit, URL: "ssh://[email protected]/repo/flake"},
"git:/repo/flake": {Type: TypeGit, URL: "git:/repo/flake"},
"git+file:///repo/flake": {Type: TypeGit, URL: "file:///repo/flake"},
"git://example.com/repo/flake?ref=unstable&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4&dir=subdir": {Type: TypeGit, URL: "git://example.com/repo/flake?dir=subdir", Ref: "unstable", Rev: "e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", Dir: "subdir"},
"git://example.com/repo/flake?ref=unstable&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4&dir=subdir": {Type: TypeGit, URL: "git://example.com/repo/flake?dir=subdir", Ref: "unstable", Rev: "e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", Dir: "subdir"},
"git://example.com/repo/flake?ref=unstable&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4&dir=subdir&lastModified=1734435836&narHash=sha256-kMBQ5PRiFLagltK0sH%2B08aiNt3zGERC2297iB6vrvlU%3D": {Type: TypeGit, URL: "git://example.com/repo/flake?dir=subdir&lastModified=1734435836&narHash=sha256-kMBQ5PRiFLagltK0sH%2B08aiNt3zGERC2297iB6vrvlU%3D", Ref: "unstable", Rev: "e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", Dir: "subdir"},

// Tarball references.
"tarball+http://example.com/flake": {Type: TypeTarball, URL: "http://example.com/flake"},
Expand All @@ -99,14 +102,15 @@ func TestParseFlakeRef(t *testing.T) {
"http://example.com/flake.tar.bz2": {Type: TypeTarball, URL: "http://example.com/flake.tar.bz2"},
"http://example.com/flake.tar.zst": {Type: TypeTarball, URL: "http://example.com/flake.tar.zst"},
"http://example.com/flake.tar?dir=subdir": {Type: TypeTarball, URL: "http://example.com/flake.tar?dir=subdir", Dir: "subdir"},
"file:///flake.zip": {Type: TypeTarball, URL: "file:///flake.zip"},
"file:///flake.tar": {Type: TypeTarball, URL: "file:///flake.tar"},
"file:///flake.tgz": {Type: TypeTarball, URL: "file:///flake.tgz"},
"file:///flake.tar.gz": {Type: TypeTarball, URL: "file:///flake.tar.gz"},
"file:///flake.tar.xz": {Type: TypeTarball, URL: "file:///flake.tar.xz"},
"file:///flake.tar.bz2": {Type: TypeTarball, URL: "file:///flake.tar.bz2"},
"file:///flake.tar.zst": {Type: TypeTarball, URL: "file:///flake.tar.zst"},
"file:///flake.tar?dir=subdir": {Type: TypeTarball, URL: "file:///flake.tar?dir=subdir", Dir: "subdir"},
"http://example.com/flake.tar?dir=subdir&lastModified=1734435836&narHash=sha256-kMBQ5PRiFLagltK0sH%2B08aiNt3zGERC2297iB6vrvlU%3D": {Type: TypeTarball, URL: "http://example.com/flake.tar?dir=subdir", Dir: "subdir", LastModified: 1734435836, NARHash: "sha256-kMBQ5PRiFLagltK0sH+08aiNt3zGERC2297iB6vrvlU="},
"file:///flake.zip": {Type: TypeTarball, URL: "file:///flake.zip"},
"file:///flake.tar": {Type: TypeTarball, URL: "file:///flake.tar"},
"file:///flake.tgz": {Type: TypeTarball, URL: "file:///flake.tgz"},
"file:///flake.tar.gz": {Type: TypeTarball, URL: "file:///flake.tar.gz"},
"file:///flake.tar.xz": {Type: TypeTarball, URL: "file:///flake.tar.xz"},
"file:///flake.tar.bz2": {Type: TypeTarball, URL: "file:///flake.tar.bz2"},
"file:///flake.tar.zst": {Type: TypeTarball, URL: "file:///flake.tar.zst"},
"file:///flake.tar?dir=subdir": {Type: TypeTarball, URL: "file:///flake.tar?dir=subdir", Dir: "subdir"},

// File URL references.
"file+file:///flake": {Type: TypeFile, URL: "file:///flake"},
Expand Down Expand Up @@ -403,5 +407,5 @@ func TestBuildQueryString(t *testing.T) {
// directives).
var elems []string
elems = append(elems, "1")
buildQueryString(elems...)
appendQueryString(nil, elems...)
}
Loading