refactor(convert): preload dependency dirs via go list (goplus#650)

* refactor(convert): preload dependency dirs via go list cache

* fix(convert): seed go-list cache from configured deps

* refactor(convert): remove xgomod from dep loader

* refactor(convert): remove BuildMod and strict go list deps preload

* refactor(convert): simplify dep list preload and cover patterns

* fix(convert): include go list output in preload errors

Author: luoliwoshang

cl/internal/convert/deps.go

@@ -1,35 +1,39 @@
 package convert
 
 import (
+	"errors"
 	"fmt"
 	"go/token"
 	"go/types"
 	"log"
+	"os/exec"
 	"path"
 	"path/filepath"
 	"strings"
 
 	"github.com/goplus/gogen"
 	llcppg "github.com/goplus/llcppg/config"
-	"github.com/goplus/mod/xgomod"
 )
 
-type Module = xgomod.Module
-
 type PkgDepLoader struct {
-	module   *xgomod.Module
+	root     string
 	pkg      *gogen.Package
 	pkgCache map[string]*PkgInfo // pkgPath -> *PkgInfo
+	pkgs     map[string]string   // pkgPath -> pkgDir
 	regCache map[string]struct{} // pkgPath
 }
 
-func NewPkgDepLoader(mod *xgomod.Module, pkg *gogen.Package) *PkgDepLoader {
-	return &PkgDepLoader{
-		module:   mod,
+func NewPkgDepLoader(root string, pkg *gogen.Package, deps []string) (*PkgDepLoader, error) {
+	ret := &PkgDepLoader{
+		root:     root,
 		pkg:      pkg,
 		pkgCache: make(map[string]*PkgInfo),
 		regCache: make(map[string]struct{}),
 	}
+	if err := ret.loadPkgDirs(deps); err != nil {
+		return nil, err
+	}
+	return ret, nil
 }
 
 // for current package & dependent packages
@@ -83,12 +87,11 @@ func (pm *PkgDepLoader) Import(pkgPath string) (*PkgInfo, error) {
 		return pkg, nil
 	}
 
-	pkg, err := pm.module.Lookup(pkgPath)
-	if err != nil {
-		return nil, err
+	pkgDir := pm.pkgs[pkgPath]
+	if pkgDir == "" {
+		return nil, fmt.Errorf("%w: go list cache has no dir for package %q", llcppg.ErrConfig, pkgPath)
 	}
-
-	pkgDir, err := filepath.Abs(pkg.Dir)
+	pkgDir, err := filepath.Abs(pkgDir)
 	if err != nil {
 		return nil, err
 	}
@@ -121,6 +124,61 @@ func (pm *PkgDepLoader) Import(pkgPath string) (*PkgInfo, error) {
 	return newPkg, nil
 }
 
+func (pm *PkgDepLoader) loadPkgDirs(deps []string) error {
+	args := []string{"list", "-deps", "-f={{.ImportPath}}={{.Dir}}"}
+
+	// Warm the go-list cache with explicit dependency patterns.
+	//
+	// We intentionally avoid relying on `go list ... all` when deps are provided:
+	// `all` only includes packages reachable from packages in the current main module.
+	// At this stage, we may have finished `go get`, but generated code has not been
+	// written yet, so those dependencies are not necessarily reachable by imports.
+	// In that case, `all` can miss entries such as github.com/goplus/lib/c.
+	//
+	// So we normalize configured deps first (for example, c -> github.com/goplus/lib/c
+	// and pkg@version -> pkg), de-duplicate them, and pass them as explicit patterns
+	// to get stable ImportPath -> Dir mappings for later dependency loading.
+	patterns := listPatterns(deps)
+	args = append(args, patterns...)
+
+	data, err := runGoCommand(pm.root, args...)
+	if err != nil {
+		return errors.New("go " + strings.Join(args, " ") + ": " + err.Error() + ": " + strings.TrimSpace(string(data)))
+	}
+	pm.pkgs = make(map[string]string)
+	for _, line := range strings.Split(string(data), "\n") {
+		parts := strings.SplitN(line, "=", 2)
+		if len(parts) == 2 && parts[0] != "" {
+			pm.pkgs[parts[0]] = parts[1]
+		}
+	}
+	return nil
+}
+
+func listPatterns(deps []string) []string {
+	seen := make(map[string]struct{})
+	patterns := make([]string, 0, len(deps))
+	for _, dep := range deps {
+		dep, _ = IsDepStd(dep)
+		dep, _ = splitPkgPath(dep)
+		if _, ok := seen[dep]; ok {
+			continue
+		}
+		seen[dep] = struct{}{}
+		patterns = append(patterns, dep)
+	}
+	if len(patterns) == 0 {
+		patterns = append(patterns, "all")
+	}
+	return patterns
+}
+
+func runGoCommand(root string, args ...string) ([]byte, error) {
+	cmd := exec.Command("go", args...)
+	cmd.Dir = root
+	return cmd.CombinedOutput()
+}
+
 func splitPkgPath(pkgPath string) (string, string) {
 	parts := strings.Split(pkgPath, "@")
 	if len(parts) == 1 {

cl/internal/convert/deps_test.go

@@ -0,0 +1,32 @@
+package convert
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestListPatterns_NormalizeAndDedup(t *testing.T) {
+	deps := []string{
+		"c",
+		"github.com/goplus/lib/c",
+		"github.com/goplus/lib/c@v0.3.1",
+		"github.com/goplus/llpkg/libxml2@v1.0.1",
+		"github.com/goplus/llpkg/libxml2",
+	}
+	got := listPatterns(deps)
+	want := []string{
+		"github.com/goplus/lib/c",
+		"github.com/goplus/llpkg/libxml2",
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("listPatterns() mismatch:\nwant: %#v\ngot:  %#v", want, got)
+	}
+}
+
+func TestListPatterns_EmptyDepsFallbackAll(t *testing.T) {
+	got := listPatterns(nil)
+	want := []string{"all"}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("listPatterns() mismatch:\nwant: %#v\ngot:  %#v", want, got)
+	}
+}

cl/internal/convert/package.go

@@ -13,7 +13,6 @@ import (
 	"github.com/goplus/llcppg/cl/nc"
 	"github.com/goplus/llcppg/internal/name"
 	ctoken "github.com/goplus/llcppg/token"
-	"github.com/goplus/mod/xgomod"
 )
 
 // In Processing Package
@@ -83,14 +82,12 @@ func NewPackage(pnc nc.NodeConverter, config *PackageConfig) (*Package, error) {
 		config.Deps = append([]string{"c"}, config.Deps...)
 	}
 
-	mod, err := xgomod.Load(config.OutputDir)
-	if err != nil {
-		return nil, fmt.Errorf("failed to load mod: %w", err)
-	}
-
 	p.PkgInfo = NewPkgInfo(config.PkgPath, config.Deps, config.Pubs)
 
-	pkgManager := NewPkgDepLoader(mod, p.p)
+	pkgManager, err := NewPkgDepLoader(config.OutputDir, p.p, config.Deps)
+	if err != nil {
+		return nil, fmt.Errorf("failed to preload package dirs: %w", err)
+	}
 	err = pkgManager.InitDeps(p.PkgInfo)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init deps: %w", err)

cl/internal/convert/package_test.go

@@ -16,7 +16,6 @@ import (
 	llcppg "github.com/goplus/llcppg/config"
 	"github.com/goplus/llcppg/internal/name"
 	"github.com/goplus/llcppg/token"
-	"github.com/goplus/mod/xgomod"
 )
 
 var dir string
@@ -1972,21 +1971,23 @@ func TestHeaderFileToGo(t *testing.T) {
 
 func TestImport(t *testing.T) {
 	t.Run("invalid include path", func(t *testing.T) {
-		p := &convert.Package{}
 		genPkg := gogen.NewPackage(".", "include", nil)
-		mod, err := xgomod.Load(".")
-		if err != nil {
-			t.Fatal(err)
-		}
 		deps := []string{
 			"github.com/goplus/llcppg/cl/internal/convert/testdata/invalidpath",
 			"github.com/goplus/llcppg/cl/internal/convert/testdata/partfinddep",
 		}
-		p.PkgInfo = convert.NewPkgInfo(".", deps, nil)
-		loader := convert.NewPkgDepLoader(mod, genPkg)
-		depPkgs, err := loader.LoadDeps(p.PkgInfo)
-		p.PkgInfo.Deps = depPkgs
-		if err != nil && !errors.Is(err, llcppg.ErrConfig) {
+		_, err := convert.NewPkgDepLoader(".", genPkg, deps)
+		if err == nil {
+			t.Fatal("expected error")
+		}
+	})
+	t.Run("unknown import path", func(t *testing.T) {
+		genPkg := gogen.NewPackage(".", "include", nil)
+		deps := []string{
+			"github.com/goplus/llcppg/cl/internal/convert/testdata/cjson",
+		}
+		loader, err := convert.NewPkgDepLoader(".", genPkg, deps)
+		if err != nil {
 			t.Fatal(err)
 		}
 		_, err = loader.Import("github.com/goplus/invalidpkg")

go.mod

@@ -6,8 +6,5 @@ require (
 	github.com/goplus/gogen v1.20.2
 	github.com/goplus/lib v0.3.1
 	github.com/goplus/llgo v0.12.1
-	github.com/goplus/mod v0.19.1
 	github.com/qiniu/x v1.16.0
 )
-
-require golang.org/x/mod v0.27.0 // indirect

go.sum

@@ -4,9 +4,5 @@ github.com/goplus/lib v0.3.1 h1:Xws4DBVvgOMu58awqB972wtvTacDbk3nqcbHjdx9KSg=
 github.com/goplus/lib v0.3.1/go.mod h1:SgJv3oPqLLHCu0gcL46ejOP3x7/2ry2Jtxu7ta32kp0=
 github.com/goplus/llgo v0.12.1 h1:LOlBxzDfano8lfeIPvNYevXbP1+aI3Ifa8JcAvOhT+A=
 github.com/goplus/llgo v0.12.1/go.mod h1:gsOG7aJtVm8Nf2uqVWAHhklt2toZMXwxItGgLxJ9gQY=
-github.com/goplus/mod v0.19.1 h1:quh1tEqhqT1dRKlWec3VNu634IqmaEF7TJaPR4LqMEs=
-github.com/goplus/mod v0.19.1/go.mod h1:4HcnqpUHXFHWRTJXQgiHrFbvy0j3K/OEzpFh5KFYLRM=
 github.com/qiniu/x v1.16.0 h1:W2VOecyIT3Uxwjm6vJinUR7G3gpwgUgHZA9OpeHArdE=
 github.com/qiniu/x v1.16.0/go.mod h1:AiovSOCaRijaf3fj+0CBOpR1457pn24b0Vdb1JpwhII=
-golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
-golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc=