Skip to content

Commit fae5a9a

Browse files
smowtonsmowton
authored
Merge pull request github#11268 from grddev/accelerate-go-list-option
Enable accelerated go-extractor opt-in using 'go list -deps'
2 parents e6c5975 + 1e01049 commit fae5a9a

File tree

2 files changed

+109
-28
lines changed

2 files changed

+109
-28
lines changed

‎go/extractor/extractor.go

Lines changed: 28 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -103,29 +103,29 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
103103
extractUniverseScope()
104104
log.Println("Done extracting universe scope.")
105105

106-
// a map of package path to package root directory (currently the module root or the source directory)
107-
pkgRoots := make(map[string]string)
108-
// a map of package path to source code directory
109-
pkgDirs := make(map[string]string)
106+
// a map of package path to source directory and module root directory
107+
pkgInfos := make(map[string]util.PkgInfo)
110108
// root directories of packages that we want to extract
111109
wantedRoots := make(map[string]bool)
112110

111+
if os.Getenv("CODEQL_EXTRACTOR_GO_FAST_PACKAGE_INFO") != "" {
112+
log.Printf("Running go list to resolve package and module directories.")
113+
// get all packages information
114+
pkgInfos, err = util.GetPkgsInfo(patterns, true, modFlags...)
115+
if err != nil {
116+
log.Fatalf("Error getting dependency package or module directories: %v.", err)
117+
}
118+
log.Printf("Done running go list deps: resolved %d packages.", len(pkgInfos))
119+
}
120+
113121
// Do a post-order traversal and extract the package scope of each package
114122
packages.Visit(pkgs, func(pkg *packages.Package) bool {
115123
return true
116124
}, func(pkg *packages.Package) {
117125
log.Printf("Processing package %s.", pkg.PkgPath)
118126

119-
if _, ok := pkgRoots[pkg.PkgPath]; !ok {
120-
mdir := util.GetModDir(pkg.PkgPath, modFlags...)
121-
pdir := util.GetPkgDir(pkg.PkgPath, modFlags...)
122-
// GetModDir returns the empty string if the module directory cannot be determined, e.g. if the package
123-
// is not using modules. If this is the case, fall back to the package directory
124-
if mdir == "" {
125-
mdir = pdir
126-
}
127-
pkgRoots[pkg.PkgPath] = mdir
128-
pkgDirs[pkg.PkgPath] = pdir
127+
if _, ok := pkgInfos[pkg.PkgPath]; !ok {
128+
pkgInfos[pkg.PkgPath] = util.GetPkgInfo(pkg.PkgPath, modFlags...)
129129
}
130130

131131
log.Printf("Extracting types for package %s.", pkg.PkgPath)
@@ -152,11 +152,14 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
152152
})
153153

154154
for _, pkg := range pkgs {
155-
if pkgRoots[pkg.PkgPath] == "" {
155+
pkgInfo, ok := pkgInfos[pkg.PkgPath]
156+
if !ok || pkgInfo.PkgDir == "" {
156157
log.Fatalf("Unable to get a source directory for input package %s.", pkg.PkgPath)
157158
}
158-
wantedRoots[pkgRoots[pkg.PkgPath]] = true
159-
wantedRoots[pkgDirs[pkg.PkgPath]] = true
159+
wantedRoots[pkgInfo.PkgDir] = true
160+
if pkgInfo.ModDir != "" {
161+
wantedRoots[pkgInfo.ModDir] = true
162+
}
160163
}
161164

162165
log.Println("Done processing dependencies.")
@@ -174,16 +177,21 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
174177
return true
175178
}, func(pkg *packages.Package) {
176179
for root, _ := range wantedRoots {
177-
relDir, err := filepath.Rel(root, pkgDirs[pkg.PkgPath])
180+
pkgInfo := pkgInfos[pkg.PkgPath]
181+
relDir, err := filepath.Rel(root, pkgInfo.PkgDir)
178182
if err != nil || noExtractRe.MatchString(relDir) {
179183
// if the path can't be made relative or matches the noExtract regexp skip it
180184
continue
181185
}
182186

183187
extraction.extractPackage(pkg)
184188

185-
if pkgRoots[pkg.PkgPath] != "" {
186-
modPath := filepath.Join(pkgRoots[pkg.PkgPath], "go.mod")
189+
modDir := pkgInfo.ModDir
190+
if modDir == "" {
191+
modDir = pkgInfo.PkgDir
192+
}
193+
if modDir != "" {
194+
modPath := filepath.Join(modDir, "go.mod")
187195
if util.FileExists(modPath) {
188196
log.Printf("Extracting %s", modPath)
189197
start := time.Now()

‎go/extractor/util/util.go

Lines changed: 81 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package util
22

33
import (
4+
"encoding/json"
45
"errors"
6+
"io"
57
"log"
68
"os"
79
"os/exec"
@@ -31,13 +33,13 @@ func Getenv(key string, aliases ...string) string {
3133

3234
// runGoList is a helper function for running go list with format `format` and flags `flags` on
3335
// package `pkgpath`.
34-
func runGoList(format string, pkgpath string, flags ...string) (string, error) {
35-
return runGoListWithEnv(format, pkgpath, nil, flags...)
36+
func runGoList(format string, patterns []string, flags ...string) (string, error) {
37+
return runGoListWithEnv(format, patterns, nil, flags...)
3638
}
3739

38-
func runGoListWithEnv(format string, pkgpath string, additionalEnv []string, flags ...string) (string, error) {
40+
func runGoListWithEnv(format string, patterns []string, additionalEnv []string, flags ...string) (string, error) {
3941
args := append([]string{"list", "-e", "-f", format}, flags...)
40-
args = append(args, pkgpath)
42+
args = append(args, patterns...)
4143
cmd := exec.Command("go", args...)
4244
cmd.Env = append(os.Environ(), additionalEnv...)
4345
out, err := cmd.Output()
@@ -54,18 +56,89 @@ func runGoListWithEnv(format string, pkgpath string, additionalEnv []string, fla
5456
return strings.TrimSpace(string(out)), nil
5557
}
5658

59+
// PkgInfo holds package directory and module directory (if any) for a package
60+
type PkgInfo struct {
61+
PkgDir string // the directory directly containing source code of this package
62+
ModDir string // the module directory containing this package, empty if not a module
63+
}
64+
65+
// GetPkgsInfo gets the absolute module and package root directories for the packages matched by the
66+
// patterns `patterns`. It passes to `go list` the flags specified by `flags`. If `includingDeps`
67+
// is true, all dependencies will also be included.
68+
func GetPkgsInfo(patterns []string, includingDeps bool, flags ...string) (map[string]PkgInfo, error) {
69+
// enable module mode so that we can find a module root if it exists, even if go module support is
70+
// disabled by a build
71+
if includingDeps {
72+
// the flag `-deps` causes all dependencies to be retrieved
73+
flags = append(flags, "-deps")
74+
}
75+
76+
// using -json overrides -f format
77+
output, err := runGoList("", patterns, append(flags, "-json")...)
78+
if err != nil {
79+
return nil, err
80+
}
81+
82+
// the output of `go list -json` is a stream of json object
83+
type goListPkgInfo struct {
84+
ImportPath string
85+
Dir string
86+
Module *struct {
87+
Dir string
88+
}
89+
}
90+
pkgInfoMapping := make(map[string]PkgInfo)
91+
streamDecoder := json.NewDecoder(strings.NewReader(output))
92+
for {
93+
var pkgInfo goListPkgInfo
94+
decErr := streamDecoder.Decode(&pkgInfo)
95+
if decErr == io.EOF {
96+
break
97+
}
98+
if decErr != nil {
99+
log.Printf("Error decoding output of go list -json: %s", err.Error())
100+
return nil, decErr
101+
}
102+
pkgAbsDir, err := filepath.Abs(pkgInfo.Dir)
103+
if err != nil {
104+
log.Printf("Unable to make package dir %s absolute: %s", pkgInfo.Dir, err.Error())
105+
}
106+
var modAbsDir string
107+
if pkgInfo.Module != nil {
108+
modAbsDir, err = filepath.Abs(pkgInfo.Module.Dir)
109+
if err != nil {
110+
log.Printf("Unable to make module dir %s absolute: %s", pkgInfo.Module.Dir, err.Error())
111+
}
112+
}
113+
pkgInfoMapping[pkgInfo.ImportPath] = PkgInfo{
114+
PkgDir: pkgAbsDir,
115+
ModDir: modAbsDir,
116+
}
117+
}
118+
return pkgInfoMapping, nil
119+
}
120+
121+
// GetPkgInfo fills the package info structure for the specified package path.
122+
// It passes the `go list` the flags specified by `flags`.
123+
func GetPkgInfo(pkgpath string, flags ...string) PkgInfo {
124+
return PkgInfo{
125+
PkgDir: GetPkgDir(pkgpath, flags...),
126+
ModDir: GetModDir(pkgpath, flags...),
127+
}
128+
}
129+
57130
// GetModDir gets the absolute directory of the module containing the package with path
58131
// `pkgpath`. It passes the `go list` the flags specified by `flags`.
59132
func GetModDir(pkgpath string, flags ...string) string {
60133
// enable module mode so that we can find a module root if it exists, even if go module support is
61134
// disabled by a build
62-
mod, err := runGoListWithEnv("{{.Module}}", pkgpath, []string{"GO111MODULE=on"}, flags...)
135+
mod, err := runGoListWithEnv("{{.Module}}", []string{pkgpath}, []string{"GO111MODULE=on"}, flags...)
63136
if err != nil || mod == "<nil>" {
64137
// if the command errors or modules aren't being used, return the empty string
65138
return ""
66139
}
67140

68-
modDir, err := runGoListWithEnv("{{.Module.Dir}}", pkgpath, []string{"GO111MODULE=on"}, flags...)
141+
modDir, err := runGoListWithEnv("{{.Module.Dir}}", []string{pkgpath}, []string{"GO111MODULE=on"}, flags...)
69142
if err != nil {
70143
return ""
71144
}
@@ -81,7 +154,7 @@ func GetModDir(pkgpath string, flags ...string) string {
81154
// GetPkgDir gets the absolute directory containing the package with path `pkgpath`. It passes the
82155
// `go list` command the flags specified by `flags`.
83156
func GetPkgDir(pkgpath string, flags ...string) string {
84-
pkgDir, err := runGoList("{{.Dir}}", pkgpath, flags...)
157+
pkgDir, err := runGoList("{{.Dir}}", []string{pkgpath}, flags...)
85158
if err != nil {
86159
return ""
87160
}
@@ -97,7 +170,7 @@ func GetPkgDir(pkgpath string, flags ...string) string {
97170
// DepErrors checks there are any errors resolving dependencies for `pkgpath`. It passes the `go
98171
// list` command the flags specified by `flags`.
99172
func DepErrors(pkgpath string, flags ...string) bool {
100-
out, err := runGoList("{{if .DepsErrors}}{{else}}error{{end}}", pkgpath, flags...)
173+
out, err := runGoList("{{if .DepsErrors}}{{else}}error{{end}}", []string{pkgpath}, flags...)
101174
if err != nil {
102175
// if go list failed, assume dependencies are broken
103176
return false

0 commit comments

Comments
 (0)