internal/searcher,lock: use /v2/resolve endpoint (#1790)

Add a `DEVBOX_FEATURE_RESOLVE_V2` feature flag that enables the new
`/v2/resolve` endpoint when resolving packages. The new endpoint unlocks
a couple of improvements:

- We no longer need to call `nix store path-from-hash-part` (which
  incurs a network request) for every `fetchClosure` package. The full
  store path is returned directly by the API.
- The resolved package response includes store paths for all known
  outputs, allowing us to support `fetchClosure` for non-default outputs
  as well.
- The new endpoint returns a flake installable rather than a nixpkgs
  commit hash and attribute path. This gives more flexibility, allowing
  the API to potentially return packages from sources other than nixpkgs.

Author: gcurtis

internal/boxcli/featureflag/resolvev2.go

@@ -0,0 +1,4 @@
+package featureflag
+
+// ResolveV2 uses the /v2/resolve endpoint when resolving packages.
+var ResolveV2 = disable("RESOLVE_V2")

internal/lock/resolve.go

@@ -16,6 +16,7 @@ import (
 	"go.jetpack.io/devbox/internal/debug"
 	"go.jetpack.io/devbox/internal/devpkg/pkgtype"
 	"go.jetpack.io/devbox/internal/nix"
+	"go.jetpack.io/devbox/internal/redact"
 	"go.jetpack.io/devbox/internal/searcher"
 	"golang.org/x/sync/errgroup"
 )
@@ -42,6 +43,9 @@ func (f *File) FetchResolvedPackage(pkg string) (*Package, error) {
 			Version:  ref.Version,
 		}, nil
 	}
+	if featureflag.ResolveV2.Enabled() {
+		return resolveV2(context.TODO(), name, version)
+	}
 
 	packageVersion, err := searcher.Client().Resolve(name, version)
 	if err != nil {
@@ -55,9 +59,9 @@ func (f *File) FetchResolvedPackage(pkg string) (*Package, error) {
 			return nil, err
 		}
 	}
-	packageInfo, err := selectForSystem(packageVersion)
+	packageInfo, err := selectForSystem(packageVersion.Systems)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("no systems found for package %q", name)
 	}
 
 	if len(packageInfo.AttrPaths) == 0 {
@@ -78,17 +82,45 @@ func (f *File) FetchResolvedPackage(pkg string) (*Package, error) {
 	}, nil
 }
 
-func selectForSystem(pkg *searcher.PackageVersion) (searcher.PackageInfo, error) {
-	if pi, ok := pkg.Systems[nix.System()]; ok {
-		return pi, nil
+func resolveV2(ctx context.Context, name, version string) (*Package, error) {
+	resolved, err := searcher.Client().ResolveV2(ctx, name, version)
+	if errors.Is(err, searcher.ErrNotFound) {
+		return nil, redact.Errorf("%s@%s: %w", name, version, nix.ErrPackageNotFound)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	// /v2/resolve never returns a success with no systems.
+	sysPkg, _ := selectForSystem(resolved.Systems)
+	pkg := &Package{
+		LastModified: sysPkg.LastUpdated.Format(time.RFC3339),
+		Resolved:     sysPkg.FlakeInstallable.String(),
+		Source:       devboxSearchSource,
+		Version:      resolved.Version,
+		Systems:      make(map[string]*SystemInfo, len(resolved.Systems)),
 	}
-	if pi, ok := pkg.Systems["x86_64-linux"]; ok {
-		return pi, nil
+	for sys, info := range resolved.Systems {
+		if len(info.Outputs) != 0 {
+			pkg.Systems[sys] = &SystemInfo{
+				StorePath: info.Outputs[0].Path,
+			}
+		}
+	}
+	return pkg, nil
+}
+
+func selectForSystem[V any](systems map[string]V) (v V, err error) {
+	if v, ok := systems[nix.System()]; ok {
+		return v, nil
+	}
+	if v, ok := systems["x86_64-linux"]; ok {
+		return v, nil
 	}
-	for _, v := range pkg.Systems {
+	for _, v := range systems {
 		return v, nil
 	}
-	return searcher.PackageInfo{}, fmt.Errorf("no systems found for package %q", pkg.Name)
+	return v, redact.Errorf("no systems found")
 }
 
 func buildLockSystemInfos(pkg *searcher.PackageVersion) (map[string]*SystemInfo, error) {

internal/searcher/client.go

@@ -4,6 +4,7 @@
 package searcher
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -12,6 +13,7 @@ import (
 
 	"github.com/pkg/errors"
 	"go.jetpack.io/devbox/internal/envir"
+	"go.jetpack.io/devbox/internal/redact"
 )
 
 const searchAPIEndpoint = "https://search.devbox.sh"
@@ -39,7 +41,7 @@ func (c *client) Search(query string) (*SearchResults, error) {
 	}
 	searchURL := endpoint + "?q=" + url.QueryEscape(query)
 
-	return execGet[SearchResults](searchURL)
+	return execGet[SearchResults](context.TODO(), searchURL)
 }
 
 // Resolve calls the /resolve endpoint of the search service. This returns
@@ -57,22 +59,57 @@ func (c *client) Resolve(name, version string) (*PackageVersion, error) {
 		"?name=" + url.QueryEscape(name) +
 		"&version=" + url.QueryEscape(version)
 
-	return execGet[PackageVersion](searchURL)
+	return execGet[PackageVersion](context.TODO(), searchURL)
 }
 
-func execGet[T any](url string) (*T, error) {
-	response, err := http.Get(url)
+// Resolve calls the /resolve endpoint of the search service. This returns
+// the latest version of the package that matches the version constraint.
+func (c *client) ResolveV2(ctx context.Context, name, version string) (*ResolveResponse, error) {
+	if name == "" {
+		return nil, redact.Errorf("name is empty")
+	}
+	if version == "" {
+		return nil, redact.Errorf("version is empty")
+	}
+
+	endpoint, err := url.JoinPath(c.host, "v2/resolve")
 	if err != nil {
-		return nil, err
+		return nil, redact.Errorf("invalid search endpoint host %q: %w", redact.Safe(c.host), redact.Safe(err))
+	}
+	searchURL := endpoint +
+		"?name=" + url.QueryEscape(name) +
+		"&version=" + url.QueryEscape(version)
+
+	return execGet[ResolveResponse](ctx, searchURL)
+}
+
+func execGet[T any](ctx context.Context, url string) (*T, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, redact.Errorf("GET %s: %w", redact.Safe(url), redact.Safe(err))
+	}
+	response, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, redact.Errorf("GET %s: %w", redact.Safe(url), redact.Safe(err))
 	}
 	defer response.Body.Close()
 	data, err := io.ReadAll(response.Body)
 	if err != nil {
-		return nil, err
+		return nil, redact.Errorf("GET %s: read respoonse body: %w", redact.Safe(url), redact.Safe(err))
 	}
 	if response.StatusCode == http.StatusNotFound {
 		return nil, ErrNotFound
 	}
+	if response.StatusCode >= 400 {
+		return nil, redact.Errorf("GET %s: unexpected status code %s: %s",
+			redact.Safe(url),
+			redact.Safe(response.Status),
+			redact.Safe(data),
+		)
+	}
 	var result T
-	return &result, json.Unmarshal(data, &result)
+	if err := json.Unmarshal(data, &result); err != nil {
+		return nil, redact.Errorf("GET %s: unmarshal response JSON: %w", redact.Safe(url), redact.Safe(err))
+	}
+	return &result, nil
 }

internal/searcher/model.go

@@ -3,6 +3,12 @@
 
 package searcher
 
+import (
+	"time"
+
+	"go.jetpack.io/devbox/nix/flake"
+)
+
 type SearchResults struct {
 	NumResults int       `json:"num_results"`
 	Packages   []Package `json:"packages,omitempty"`
@@ -35,3 +41,56 @@ type PackageInfo struct {
 	Version      string   `json:"version"`
 	Summary      string   `json:"summary"`
 }
+
+// ResolveResponse is a response from the /v2/resolve endpoint.
+type ResolveResponse struct {
+	// Name is the resolved name of the package. For packages that are
+	// identifiable by multiple names or attribute paths, this is the
+	// "canonical" name.
+	Name string `json:"name"`
+
+	// Version is the resolved package version.
+	Version string `json:"version"`
+
+	// Summary is a short package description.
+	Summary string `json:"summary,omitempty"`
+
+	// Systems contains information about the package that can vary across
+	// systems. It will always have at least one system. The keys match a
+	// Nix system identifier (aarch64-darwin, x86_64-linux, etc.).
+	Systems map[string]struct {
+		// FlakeInstallable is a Nix installable that specifies how to
+		// install the resolved package version.
+		//
+		// [Nix installable]: https://nixos.org/manual/nix/stable/command-ref/new-cli/nix#installables
+		FlakeInstallable flake.Installable `json:"flake_installable"`
+
+		// LastUpdated is the timestamp of the most recent change to the
+		// package.
+		LastUpdated time.Time `json:"last_updated"`
+
+		// Outputs provides additional information about the Nix store
+		// paths that this package installs. This field is not available
+		// for some (especially older) packages.
+		Outputs []struct {
+			// Name is the output's name. Nix appends the name to
+			// the output's store path unless it's the default name
+			// of "out". Output names can be anything, but
+			// conventionally they follow the various "make install"
+			// directories such as "bin", "lib", "src", "man", etc.
+			Name string `json:"name,omitempty"`
+
+			// Path is the absolute store path (with the /nix/store/
+			// prefix) of the output.
+			Path string `json:"path,omitempty"`
+
+			// Default indicates if Nix installs this output by
+			// default.
+			Default bool `json:"default,omitempty"`
+
+			// NAR is set to the package's NAR archive URL when the
+			// output exists in the cache.nixos.org binary cache.
+			NAR string `json:"nar,omitempty"`
+		} `json:"outputs,omitempty"`
+	} `json:"systems"`
+}

nix/flake/flakeref.go

@@ -456,11 +456,11 @@ const (
 // [Nix manual]: https://nixos.org/manual/nix/unstable/command-ref/new-cli/nix#installables
 type Installable struct {
 	// Ref is the flake reference portion of the installable.
-	Ref Ref
+	Ref Ref `json:"ref,omitempty"`
 
 	// AttrPath is an attribute path of the flake, encoded as a URL
 	// fragment.
-	AttrPath string
+	AttrPath string `json:"attr_path,omitempty"`
 
 	// Outputs is the installable's output spec, which is a comma-separated
 	// list of package outputs to install. The outputs spec is anything
@@ -474,7 +474,7 @@ type Installable struct {
 	// ParseInstallable cleans the list of outputs by removing empty
 	// elements and sorting the results. Lists containing a "*" are
 	// simplified to a single "*".
-	Outputs string
+	Outputs string `json:"outputs,omitempty"`
 }
 
 // ParseInstallable parses a flake installable. The raw string must contain