Accelerating go-extractor by using 'go list -deps'

Resurrect github/codeql-go#554, but behind an environment variable as to avoid the broken builds noted in github#9304, but still allowing some people to opt in to the much faster approach.

Author: grddev

go/extractor/extractor.go

@@ -108,6 +108,16 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 	// root directories of packages that we want to extract
 	wantedRoots := make(map[string]bool)
 
+	if os.Getenv("CODEQL_EXTRACTOR_GO_FAST_PACKAGE_INFO") != "" {
+		log.Printf("Running go list to resolve package and module directories.")
+		// get all packages information
+		pkgInfos, err = util.GetPkgsInfo(patterns, true, modFlags...)
+		if err != nil {
+			log.Fatalf("Error getting dependency package or module directories: %v.", err)
+		}
+		log.Printf("Done running go list deps: resolved %d packages.", len(pkgInfos))
+	}
+
 	// Do a post-order traversal and extract the package scope of each package
 	packages.Visit(pkgs, func(pkg *packages.Package) bool {
 		return true

go/extractor/util/util.go

@@ -1,7 +1,9 @@
 package util
 
 import (
+	"encoding/json"
 	"errors"
+	"io"
 	"log"
 	"os"
 	"os/exec"
@@ -31,13 +33,13 @@ func Getenv(key string, aliases ...string) string {
 
 // runGoList is a helper function for running go list with format `format` and flags `flags` on
 // package `pkgpath`.
-func runGoList(format string, pkgpath string, flags ...string) (string, error) {
-	return runGoListWithEnv(format, pkgpath, nil, flags...)
+func runGoList(format string, patterns []string, flags ...string) (string, error) {
+	return runGoListWithEnv(format, patterns, nil, flags...)
 }
 
-func runGoListWithEnv(format string, pkgpath string, additionalEnv []string, flags ...string) (string, error) {
+func runGoListWithEnv(format string, patterns []string, additionalEnv []string, flags ...string) (string, error) {
 	args := append([]string{"list", "-e", "-f", format}, flags...)
-	args = append(args, pkgpath)
+	args = append(args, patterns...)
 	cmd := exec.Command("go", args...)
 	cmd.Env = append(os.Environ(), additionalEnv...)
 	out, err := cmd.Output()
@@ -60,6 +62,63 @@ type PkgInfo struct {
 	ModDir string // the module directory containing this package, empty if not a module
 }
 
+// GetPkgsInfo gets the absolute module and package root directories for the packages matched by the
+// patterns `patterns`. It passes to `go list` the flags specified by `flags`.  If `includingDeps`
+// is true, all dependencies will also be included.
+func GetPkgsInfo(patterns []string, includingDeps bool, flags ...string) (map[string]PkgInfo, error) {
+	// enable module mode so that we can find a module root if it exists, even if go module support is
+	// disabled by a build
+	if includingDeps {
+		// the flag `-deps` causes all dependencies to be retrieved
+		flags = append(flags, "-deps")
+	}
+
+	// using -json overrides -f format
+	output, err := runGoList("", patterns, append(flags, "-json")...)
+	if err != nil {
+		return nil, err
+	}
+
+	// the output of `go list -json` is a stream of json object
+	type goListPkgInfo struct {
+		ImportPath string
+		Dir        string
+		Module     *struct {
+			Dir string
+		}
+	}
+	pkgInfoMapping := make(map[string]PkgInfo)
+	streamDecoder := json.NewDecoder(strings.NewReader(output))
+	for {
+		var pkgInfo goListPkgInfo
+		decErr := streamDecoder.Decode(&pkgInfo)
+		if decErr == io.EOF {
+			break
+		}
+		if decErr != nil {
+			log.Printf("Error decoding output of go list -json: %s", err.Error())
+			return nil, decErr
+		}
+		pkgAbsDir, err := filepath.Abs(pkgInfo.Dir)
+		if err != nil {
+			log.Printf("Unable to make package dir %s absolute: %s", pkgInfo.Dir, err.Error())
+		}
+		var modAbsDir string
+		if pkgInfo.Module != nil {
+			modAbsDir, err = filepath.Abs(pkgInfo.Module.Dir)
+			if err != nil {
+				log.Printf("Unable to make module dir %s absolute: %s", pkgInfo.Module.Dir, err.Error())
+			}
+		}
+		pkgInfoMapping[pkgInfo.ImportPath] = PkgInfo{
+			PkgDir: pkgAbsDir,
+			ModDir: modAbsDir,
+		}
+	}
+	return pkgInfoMapping, nil
+}
+
+
 // GetPkgInfo fills the package info structure for the specified package path.
 // It passes the `go list` the flags specified by `flags`.
 func GetPkgInfo(pkgpath string, flags ...string) PkgInfo {
@@ -74,13 +133,13 @@ func GetPkgInfo(pkgpath string, flags ...string) PkgInfo {
 func GetModDir(pkgpath string, flags ...string) string {
 	// enable module mode so that we can find a module root if it exists, even if go module support is
 	// disabled by a build
-	mod, err := runGoListWithEnv("{{.Module}}", pkgpath, []string{"GO111MODULE=on"}, flags...)
+	mod, err := runGoListWithEnv("{{.Module}}", []string{pkgpath}, []string{"GO111MODULE=on"}, flags...)
 	if err != nil || mod == "<nil>" {
 		// if the command errors or modules aren't being used, return the empty string
 		return ""
 	}
 
-	modDir, err := runGoListWithEnv("{{.Module.Dir}}", pkgpath, []string{"GO111MODULE=on"}, flags...)
+	modDir, err := runGoListWithEnv("{{.Module.Dir}}", []string{pkgpath}, []string{"GO111MODULE=on"}, flags...)
 	if err != nil {
 		return ""
 	}
@@ -96,7 +155,7 @@ func GetModDir(pkgpath string, flags ...string) string {
 // GetPkgDir gets the absolute directory containing the package with path `pkgpath`. It passes the
 // `go list` command the flags specified by `flags`.
 func GetPkgDir(pkgpath string, flags ...string) string {
-	pkgDir, err := runGoList("{{.Dir}}", pkgpath, flags...)
+	pkgDir, err := runGoList("{{.Dir}}", []string{pkgpath}, flags...)
 	if err != nil {
 		return ""
 	}
@@ -112,7 +171,7 @@ func GetPkgDir(pkgpath string, flags ...string) string {
 // DepErrors checks there are any errors resolving dependencies for `pkgpath`. It passes the `go
 // list` command the flags specified by `flags`.
 func DepErrors(pkgpath string, flags ...string) bool {
-	out, err := runGoList("{{if .DepsErrors}}{{else}}error{{end}}", pkgpath, flags...)
+	out, err := runGoList("{{if .DepsErrors}}{{else}}error{{end}}", []string{pkgpath}, flags...)
 	if err != nil {
 		// if go list failed, assume dependencies are broken
 		return false