Skip to content
This repository was archived by the owner on Jan 5, 2023. It is now read-only.

Commit 2b07e6a

Browse files
smowtonsmowton
authored
Merge pull request #324 from sauyon/tracing
Build tracing
2 parents 4746789 + e5afd1d commit 2b07e6a

File tree

13 files changed

+266
-46
lines changed

13 files changed

+266
-46
lines changed

‎Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,11 @@ CODEQL_PLATFORM = osx64
1414
endif
1515
endif
1616

17-
CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh index.cmd index.sh)
17+
CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh index.cmd index.sh linux64 osx64 win64)
1818

1919
EXTRACTOR_PACK_OUT = build/codeql-extractor-go
2020

21-
BINARIES = go-extractor go-tokenizer go-autobuilder go-bootstrap go-gen-dbscheme
21+
BINARIES = go-extractor go-tokenizer go-autobuilder go-build-runner go-bootstrap go-gen-dbscheme
2222

2323
.PHONY: tools tools-codeql tools-codeql-full clean autoformat \
2424
tools-linux64 tools-osx64 tools-win64 check-formatting

‎change-notes/2020-06-11-build-tracing.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
lgtm,codescanning
2+
* The Go extractor now supports build tracing, allowing users to supply a build command when
3+
creating databases with the CodeQL CLI or via configuration. It currently only supports projects
4+
that use Go modules. To opt-in, set the environment variable `CODEQL_EXTRACTOR_GO_BUILD_TRACING`
5+
to `on`, or supply a build command.

‎codeql-tools/autobuild.cmd

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,12 @@ SETLOCAL EnableDelayedExpansion
44
rem Some legacy environment variables for the autobuilder.
55
set LGTM_SRC=%CD%
66

7-
type NUL && "%CODEQL_EXTRACTOR_GO_ROOT%/tools/%CODEQL_PLATFORM%/go-autobuilder.exe"
7+
if "%CODEQL_EXTRACTOR_GO_BUILD_TRACING%"=="on" (
8+
echo "Tracing enabled"
9+
type NUL && "%CODEQL_EXTRACTOR_GO_ROOT%/tools/%CODEQL_PLATFORM%/go-build-runner.exe"
10+
) else (
11+
type NUL && "%CODEQL_EXTRACTOR_GO_ROOT%/tools/%CODEQL_PLATFORM%/go-autobuilder.exe"
12+
)
813
exit /b %ERRORLEVEL%
914

1015
ENDLOCAL

‎codeql-tools/autobuild.sh

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,9 @@ fi
1111
LGTM_SRC="$(pwd)"
1212
export LGTM_SRC
1313

14-
"$CODEQL_EXTRACTOR_GO_ROOT/tools/$CODEQL_PLATFORM/go-autobuilder"
14+
if [ "${CODEQL_EXTRACTOR_GO_BUILD_TRACING:-}" == "on" ]; then
15+
echo "Tracing enabled"
16+
"$CODEQL_EXTRACTOR_GO_ROOT/tools/$CODEQL_PLATFORM/go-build-runner"
17+
else
18+
"$CODEQL_EXTRACTOR_GO_ROOT/tools/$CODEQL_PLATFORM/go-autobuilder"
19+
fi

‎codeql-tools/linux64/compiler-tracing.spec

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
**/go-autobuilder:
2+
order compiler
3+
trace no
4+
**/go:
5+
invoke ${config_dir}/go-extractor
6+
prepend --mimic
7+
prepend "${compiler}"

‎codeql-tools/osx64/compiler-tracing.spec

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
**/go-autobuilder:
2+
order compiler
3+
trace no
4+
**/go:
5+
invoke ${config_dir}/go-extractor
6+
prepend --mimic
7+
prepend "${compiler}"
8+
/usr/bin/codesign:
9+
replace yes
10+
invoke /usr/bin/env
11+
prepend /usr/bin/codesign
12+
trace no
13+
/usr/bin/pkill:
14+
replace yes
15+
invoke /usr/bin/env
16+
prepend /usr/bin/pkill
17+
trace no
18+
/usr/bin/pgrep:
19+
replace yes
20+
invoke /usr/bin/env
21+
prepend /usr/bin/pgrep
22+
trace no

‎codeql-tools/win64/compiler-tracing.spec

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
**/go-autobuilder.exe:
2+
order compiler
3+
trace no
4+
**/go.exe:
5+
invoke ${config_dir}/go-extractor.exe
6+
prepend --mimic
7+
prepend "${compiler}"

‎extractor/autobuilder/autobuilder.go

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
// Package autobuilder implements a simple system that attempts to run build commands for common
2+
// build frameworks, if the relevant files exist.
3+
package autobuilder
4+
5+
import (
6+
"log"
7+
"os"
8+
"os/exec"
9+
10+
"github.com/github/codeql-go/extractor/util"
11+
)
12+
13+
// CheckExtracted sets whether the autobuilder should check whether source files have been extracted
14+
// to the CodeQL source directory as well as whether the build command executed successfully.
15+
var CheckExtracted = false
16+
17+
// checkEmpty checks whether a directory either doesn't exist or is empty.
18+
func checkEmpty(dir string) (bool, error) {
19+
if !util.DirExists(dir) {
20+
return true, nil
21+
}
22+
23+
d, err := os.Open(dir)
24+
if err != nil {
25+
return false, err
26+
}
27+
defer d.Close()
28+
29+
names, err := d.Readdirnames(-1)
30+
if err != nil {
31+
return false, err
32+
}
33+
return len(names) == 0, nil
34+
}
35+
36+
// checkExtractorRun checks whether the CodeQL Go extractor has run, by checking if the source
37+
// archive directory is empty or not.
38+
func checkExtractorRun() bool {
39+
srcDir := os.Getenv("CODEQL_EXTRACTOR_GO_SOURCE_ARCHIVE_DIR")
40+
if srcDir != "" {
41+
empty, err := checkEmpty(srcDir)
42+
if err != nil {
43+
log.Fatalf("Unable to read source archive directory %s.", srcDir)
44+
}
45+
if empty {
46+
log.Printf("No Go code seen; continuing to try other builds.")
47+
return false
48+
}
49+
return true
50+
} else {
51+
log.Fatalf("No source directory set.\nThis binary should not be run manually; instead, use the CodeQL CLI or VSCode extension. See https://securitylab.github.com/tools/codeql.")
52+
return false
53+
}
54+
}
55+
56+
// tryBuildIfExists tries to run the command `cmd args...` if the file `buildFile` exists and is not
57+
// a directory. Returns true if the command was successful and false if not.
58+
func tryBuildIfExists(buildFile, cmd string, args ...string) bool {
59+
if util.FileExists(buildFile) {
60+
log.Printf("%s found.\n", buildFile)
61+
return tryBuild(cmd, args...)
62+
}
63+
return false
64+
}
65+
66+
// tryBuild tries to run `cmd args...`, returning true if successful and false if not.
67+
func tryBuild(cmd string, args ...string) bool {
68+
log.Printf("Trying build command %s %v", cmd, args)
69+
res := util.RunCmd(exec.Command(cmd, args...))
70+
return res && (!CheckExtracted || checkExtractorRun())
71+
}
72+
73+
// Autobuild attempts to detect build system and run the corresponding command.
74+
func Autobuild() bool {
75+
return tryBuildIfExists("Makefile", "make") ||
76+
tryBuildIfExists("makefile", "make") ||
77+
tryBuildIfExists("GNUmakefile", "make") ||
78+
tryBuildIfExists("build.ninja", "ninja") ||
79+
tryBuildIfExists("build", "./build") ||
80+
tryBuildIfExists("build.sh", "./build.sh")
81+
}

‎extractor/cli/go-autobuilder/go-autobuilder.go

Lines changed: 7 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"runtime"
1414
"strings"
1515

16+
"github.com/github/codeql-go/extractor/autobuilder"
1617
"github.com/github/codeql-go/extractor/util"
1718
)
1819

@@ -68,29 +69,10 @@ func getEnvGoSemVer() string {
6869
return "v" + goVersion[2:]
6970
}
7071

71-
func run(cmd *exec.Cmd) bool {
72-
cmd.Stdout = os.Stdout
73-
cmd.Stderr = os.Stderr
74-
in, _ := cmd.StdinPipe()
75-
err := cmd.Start()
76-
if err != nil {
77-
log.Printf("Running %s failed, continuing anyway: %s\n", cmd.Path, err.Error())
78-
return false
79-
}
80-
in.Close()
81-
err = cmd.Wait()
82-
if err != nil {
83-
log.Printf("Running %s failed, continuing anyway: %s\n", cmd.Path, err.Error())
84-
return false
85-
}
86-
87-
return true
88-
}
89-
9072
func tryBuild(buildFile, cmd string, args ...string) bool {
9173
if util.FileExists(buildFile) {
9274
log.Printf("%s found, running %s\n", buildFile, cmd)
93-
return run(exec.Command(cmd, args...))
75+
return util.RunCmd(exec.Command(cmd, args...))
9476
}
9577
return false
9678
}
@@ -209,7 +191,7 @@ func (m ModMode) argsForGoVersion(version string) []string {
209191
// addVersionToMod add a go version directive, e.g. `go 1.14` to a `go.mod` file.
210192
func addVersionToMod(goMod []byte, version string) bool {
211193
cmd := exec.Command("go", "mod", "edit", "-go="+version)
212-
return run(cmd)
194+
return util.RunCmd(cmd)
213195
}
214196

215197
// checkVendor tests to see whether a vendor directory is inconsistent according to the go frontend
@@ -422,13 +404,8 @@ func main() {
422404
inst := util.Getenv("CODEQL_EXTRACTOR_GO_BUILD_COMMAND", "LGTM_INDEX_BUILD_COMMAND")
423405
shouldInstallDependencies := false
424406
if inst == "" {
425-
// if there is a build file, run the corresponding build tool
426-
buildSucceeded := tryBuild("Makefile", "make") ||
427-
tryBuild("makefile", "make") ||
428-
tryBuild("GNUmakefile", "make") ||
429-
tryBuild("build.ninja", "ninja") ||
430-
tryBuild("build", "./build") ||
431-
tryBuild("build.sh", "./build.sh")
407+
// try to build the project
408+
buildSucceeded := autobuilder.Autobuild()
432409

433410
if !buildSucceeded {
434411
// Build failed; we'll try to install dependencies ourselves
@@ -464,7 +441,7 @@ func main() {
464441
}
465442
os.Chmod(script.Name(), 0700)
466443
log.Println("Installing dependencies using custom build command.")
467-
run(exec.Command(script.Name()))
444+
util.RunCmd(exec.Command(script.Name()))
468445
}
469446

470447
if modMode == ModVendor {
@@ -525,7 +502,7 @@ func main() {
525502
install = exec.Command("go", "get", "-v", "./...")
526503
log.Println("Installing dependencies using `go get -v ./...`.")
527504
}
528-
run(install)
505+
util.RunCmd(install)
529506
}
530507
}
531508

‎extractor/cli/go-build-runner/go-build-runner.go

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package main
2+
3+
import (
4+
"github.com/github/codeql-go/extractor/util"
5+
"log"
6+
"os"
7+
"os/exec"
8+
"path/filepath"
9+
"runtime"
10+
11+
"github.com/github/codeql-go/extractor/autobuilder"
12+
)
13+
14+
func main() {
15+
// check if a build command has successfully extracted something
16+
autobuilder.CheckExtracted = true
17+
if autobuilder.Autobuild() {
18+
return
19+
}
20+
21+
// if the autobuilder fails, invoke the extractor manually
22+
// we cannot simply call `go build` here, because the tracer is not able to trace calls made by
23+
// this binary
24+
log.Printf("No build commands succeeded, falling back to go build ./...")
25+
26+
mypath, err := os.Executable()
27+
if err != nil {
28+
log.Fatalf("Could not determine path of extractor: %v.\n", err)
29+
}
30+
extractor := filepath.Join(filepath.Dir(mypath), "go-extractor")
31+
if runtime.GOOS == "windows" {
32+
extractor = extractor + ".exe"
33+
}
34+
35+
util.RunCmd(exec.Command(extractor, "./..."))
36+
}

0 commit comments

Comments
 (0)