Merge pull request github#12341 from owen-mc/go-tools-status

Go: tools status page support

Author: owen-mc

go/extractor/cli/go-autobuilder/go-autobuilder.go

@@ -16,6 +16,7 @@ import (
 	"golang.org/x/mod/semver"
 
 	"github.com/github/codeql-go/extractor/autobuilder"
+	"github.com/github/codeql-go/extractor/diagnostics"
 	"github.com/github/codeql-go/extractor/util"
 )
 
@@ -203,7 +204,7 @@ func (m ModMode) argsForGoVersion(version string) []string {
 }
 
 // addVersionToMod add a go version directive, e.g. `go 1.14` to a `go.mod` file.
-func addVersionToMod(goMod []byte, version string) bool {
+func addVersionToMod(version string) bool {
 	cmd := exec.Command("go", "mod", "edit", "-go="+version)
 	return util.RunCmd(cmd)
 }
@@ -249,12 +250,29 @@ func main() {
 	depMode := GoGetNoModules
 	modMode := ModUnset
 	needGopath := true
+	goDirectiveFound := false
 	if _, present := os.LookupEnv("GO111MODULE"); !present {
 		os.Setenv("GO111MODULE", "auto")
 	}
 	if util.FileExists("go.mod") {
 		depMode = GoGetWithModules
 		needGopath = false
+		versionRe := regexp.MustCompile(`(?m)^go[ \t\r]+([0-9]+\.[0-9]+)$`)
+		goMod, err := ioutil.ReadFile("go.mod")
+		if err != nil {
+			log.Println("Failed to read go.mod to check for missing Go version")
+		} else {
+			matches := versionRe.FindSubmatch(goMod)
+			if matches != nil {
+				goDirectiveFound = true
+				if len(matches) > 1 {
+					goDirectiveVersion := "v" + string(matches[1])
+					if semver.Compare(goDirectiveVersion, getEnvGoSemVer()) >= 0 {
+						diagnostics.EmitNewerGoVersionNeeded()
+					}
+				}
+			}
+		}
 		log.Println("Found go.mod, enabling go modules")
 	} else if util.FileExists("Gopkg.toml") {
 		depMode = Dep
@@ -283,18 +301,15 @@ func main() {
 		// we work around this by adding an explicit go version of 1.13, which is the last version
 		// where this is not an issue
 		if depMode == GoGetWithModules {
-			goMod, err := ioutil.ReadFile("go.mod")
-			if err != nil {
-				log.Println("Failed to read go.mod to check for missing Go version")
-			} else if versionRe := regexp.MustCompile(`(?m)^go[ \t\r]+[0-9]+\.[0-9]+$`); !versionRe.Match(goMod) {
+			if !goDirectiveFound {
 				// if the go.mod does not contain a version line
 				modulesTxt, err := ioutil.ReadFile("vendor/modules.txt")
 				if err != nil {
 					log.Println("Failed to read vendor/modules.txt to check for mismatched Go version")
 				} else if explicitRe := regexp.MustCompile("(?m)^## explicit$"); !explicitRe.Match(modulesTxt) {
 					// and the modules.txt does not contain an explicit annotation
 					log.Println("Adding a version directive to the go.mod file as the modules.txt does not have explicit annotations")
-					if !addVersionToMod(goMod, "1.13") {
+					if !addVersionToMod("1.13") {
 						log.Println("Failed to add a version to the go.mod file to fix explicitly required package bug; not using vendored dependencies")
 						modMode = ModMod
 					}

go/extractor/diagnostics/diagnostics.go

@@ -0,0 +1,183 @@
+package diagnostics
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+	"time"
+)
+
+type sourceStruct struct {
+	Id            string `json:"id"`
+	Name          string `json:"name"`
+	ExtractorName string `json:"extractorName"`
+}
+
+type diagnosticSeverity string
+
+const (
+	severityError   diagnosticSeverity = "error"
+	severityWarning diagnosticSeverity = "warning"
+	severityNote    diagnosticSeverity = "note"
+)
+
+type visibilityStruct struct {
+	StatusPage      bool `json:"statusPage"`      // True if the message should be displayed on the status page (defaults to false)
+	CliSummaryTable bool `json:"cliSummaryTable"` // True if the message should be counted in the diagnostics summary table printed by `codeql database analyze` (defaults to false)
+	Telemetry       bool `json:"telemetry"`       // True if the message should be sent to telemetry (defaults to false)
+}
+
+var fullVisibility *visibilityStruct = &visibilityStruct{true, true, true}
+
+type locationStruct struct {
+	File        string `json:"file,omitempty"`
+	StartLine   int    `json:"startLine,omitempty"`
+	StartColumn int    `json:"startColumn,omitempty"`
+	EndLine     int    `json:"endLine,omitempty"`
+	EndColumn   int    `json:"endColumn,omitempty"`
+}
+
+var noLocation *locationStruct = nil
+
+type diagnostic struct {
+	Timestamp       string            `json:"timestamp"`
+	Source          sourceStruct      `json:"source"`
+	MarkdownMessage string            `json:"markdownMessage"`
+	Severity        string            `json:"severity"`
+	Visibility      *visibilityStruct `json:"visibility,omitempty"` // Use a pointer so that it is omitted if nil
+	Location        *locationStruct   `json:"location,omitempty"`   // Use a pointer so that it is omitted if nil
+}
+
+var diagnosticsEmitted, diagnosticsLimit uint = 0, 100
+var noDiagnosticDirPrinted bool = false
+
+func emitDiagnostic(sourceid, sourcename, markdownMessage string, severity diagnosticSeverity, visibility *visibilityStruct, location *locationStruct) {
+	if diagnosticsEmitted < diagnosticsLimit {
+		diagnosticsEmitted += 1
+
+		diagnosticDir := os.Getenv("CODEQL_EXTRACTOR_GO_DIAGNOSTIC_DIR")
+		if diagnosticDir == "" {
+			if !noDiagnosticDirPrinted {
+				log.Println("No diagnostic directory set, so not emitting diagnostic")
+				noDiagnosticDirPrinted = true
+			}
+			return
+		}
+
+		timestamp := time.Now().UTC().Format("2006-01-02T15:04:05.000") + "Z"
+
+		var d diagnostic
+
+		if diagnosticsEmitted < diagnosticsLimit {
+			d = diagnostic{
+				timestamp,
+				sourceStruct{sourceid, sourcename, "go"},
+				markdownMessage,
+				string(severity),
+				visibility,
+				location,
+			}
+		} else {
+			d = diagnostic{
+				timestamp,
+				sourceStruct{"go/autobuilder/diagnostic-limit-reached", "Diagnostics limit exceeded", "go"},
+				fmt.Sprintf("CodeQL has produced more than the maximum number of diagnostics. Only the first %d have been reported.", diagnosticsLimit),
+				string(severityWarning),
+				fullVisibility,
+				noLocation,
+			}
+		}
+
+		content, err := json.Marshal(d)
+		if err != nil {
+			log.Println(err)
+			return
+		}
+
+		targetFile, err := os.CreateTemp(diagnosticDir, "go-extractor.*.json")
+		if err != nil {
+			log.Println("Failed to create diagnostic file: ")
+			log.Println(err)
+			return
+		}
+		defer func() {
+			if err := targetFile.Close(); err != nil {
+				log.Println("Failed to close diagnostic file:")
+				log.Println(err)
+			}
+		}()
+
+		_, err = targetFile.Write(content)
+		if err != nil {
+			log.Println("Failed to write to diagnostic file: ")
+			log.Println(err)
+		}
+	}
+}
+
+func EmitPackageDifferentOSArchitecture(pkgPath string) {
+	emitDiagnostic(
+		"go/autobuilder/package-different-os-architecture",
+		"Package "+pkgPath+" is intended for a different OS or architecture",
+		"Make sure the `GOOS` and `GOARCH` [environment variables are correctly set](https://docs.github.com/en/actions/learn-github-actions/variables#defining-environment-variables-for-a-single-workflow). Alternatively, [change your OS and architecture](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#using-a-github-hosted-runner).",
+		severityWarning,
+		fullVisibility,
+		noLocation,
+	)
+}
+
+const maxNumPkgPaths = 5
+
+func EmitCannotFindPackages(pkgPaths []string) {
+	numPkgPaths := len(pkgPaths)
+
+	ending := "s"
+	if numPkgPaths == 1 {
+		ending = ""
+	}
+
+	numPrinted := numPkgPaths
+	truncated := false
+	if numPrinted > maxNumPkgPaths {
+		numPrinted = maxNumPkgPaths
+		truncated = true
+	}
+
+	secondLine := "`" + strings.Join(pkgPaths[0:numPrinted], "`, `") + "`"
+	if truncated {
+		secondLine += fmt.Sprintf(" and %d more", numPkgPaths-maxNumPkgPaths)
+	}
+
+	emitDiagnostic(
+		"go/autobuilder/package-not-found",
+		fmt.Sprintf("%d package%s could not be found", numPkgPaths, ending),
+		"The following packages could not be found.\n\n"+secondLine+"\n\nCheck that the paths are correct and make sure any private packages can be accessed. If any of the packages are present in the repository then you may need a [custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+		severityError,
+		fullVisibility,
+		noLocation,
+	)
+}
+
+func EmitNewerGoVersionNeeded() {
+	emitDiagnostic(
+		"go/autobuilder/newer-go-version-needed",
+		"Newer Go version needed",
+		"The detected version of Go is lower than the version specified in `go.mod`. [Install a newer version](https://github.com/actions/setup-go#basic).",
+		severityError,
+		fullVisibility,
+		noLocation,
+	)
+}
+
+func EmitGoFilesFoundButNotProcessed() {
+	emitDiagnostic(
+		"go/autobuilder/go-files-found-but-not-processed",
+		"Go files were found but not processed",
+		"[Specify a custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages) that includes one or more `go build` commands to build the `.go` files to be analyzed.",
+		severityError,
+		fullVisibility,
+		noLocation,
+	)
+}

go/extractor/extractor.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/github/codeql-go/extractor/dbscheme"
+	"github.com/github/codeql-go/extractor/diagnostics"
 	"github.com/github/codeql-go/extractor/srcarchive"
 	"github.com/github/codeql-go/extractor/trap"
 	"github.com/github/codeql-go/extractor/util"
@@ -97,6 +98,13 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 
 	if len(pkgs) == 0 {
 		log.Println("No packages found.")
+
+		wd, err := os.Getwd()
+		if err != nil {
+			log.Printf("Warning: failed to get working directory: %s\n", err.Error())
+		} else if util.FindGoFiles(wd) {
+			diagnostics.EmitGoFilesFoundButNotProcessed()
+		}
 	}
 
 	log.Println("Extracting universe scope.")
@@ -118,6 +126,8 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 		log.Printf("Done running go list deps: resolved %d packages.", len(pkgInfos))
 	}
 
+	pkgsNotFound := make([]string, 0, len(pkgs))
+
 	// Do a post-order traversal and extract the package scope of each package
 	packages.Visit(pkgs, func(pkg *packages.Package) bool {
 		return true
@@ -144,13 +154,27 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 		if len(pkg.Errors) != 0 {
 			log.Printf("Warning: encountered errors extracting package `%s`:", pkg.PkgPath)
 			for i, err := range pkg.Errors {
-				log.Printf("  %s", err.Error())
+				errString := err.Error()
+				log.Printf("  %s", errString)
+
+				if strings.Contains(errString, "build constraints exclude all Go files in ") {
+					// `err` is a NoGoError from the package cmd/go/internal/load, which we cannot access as it is internal
+					diagnostics.EmitPackageDifferentOSArchitecture(pkg.PkgPath)
+				} else if strings.Contains(errString, "cannot find package") ||
+					strings.Contains(errString, "no required module provides package") {
+					pkgsNotFound = append(pkgsNotFound, pkg.PkgPath)
+				}
 				extraction.extractError(tw, err, lbl, i)
 			}
 		}
+
 		log.Printf("Done extracting types for package %s.", pkg.PkgPath)
 	})
 
+	if len(pkgsNotFound) > 0 {
+		diagnostics.EmitCannotFindPackages(pkgsNotFound)
+	}
+
 	for _, pkg := range pkgs {
 		pkgInfo, ok := pkgInfos[pkg.PkgPath]
 		if !ok || pkgInfo.PkgDir == "" {

go/extractor/util/util.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"io"
+	"io/fs"
 	"log"
 	"os"
 	"os/exec"
@@ -281,3 +282,18 @@ func EscapeTrapSpecialChars(s string) string {
 	s = strings.ReplaceAll(s, "#", "#")
 	return s
 }
+
+func FindGoFiles(root string) bool {
+	found := false
+	filepath.WalkDir(root, func(s string, d fs.DirEntry, e error) error {
+		if e != nil {
+			return e
+		}
+		if filepath.Ext(d.Name()) == ".go" {
+			found = true
+			return filepath.SkipAll
+		}
+		return nil
+	})
+	return found
+}

go/ql/integration-tests/all-platforms/go/diagnostics/build-constraints-exclude-all-go-files/diagnostics.expected

@@ -0,0 +1,14 @@
+{
+  "markdownMessage": "Make sure the `GOOS` and `GOARCH` [environment variables are correctly set](https://docs.github.com/en/actions/learn-github-actions/variables#defining-environment-variables-for-a-single-workflow). Alternatively, [change your OS and architecture](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#using-a-github-hosted-runner).",
+  "severity": "warning",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/package-different-os-architecture",
+    "name": "Package syscall/js is intended for a different OS or architecture"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}

go/ql/integration-tests/all-platforms/go/diagnostics/build-constraints-exclude-all-go-files/test.py

@@ -0,0 +1,9 @@
+import sys
+
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+os.environ['LGTM_INDEX_IMPORT_PATH'] = "test"
+run_codeql_database_create([], lang="go", source="work", db=None)
+
+check_diagnostics()

go/ql/integration-tests/all-platforms/go/diagnostics/build-constraints-exclude-all-go-files/work/go.mod

@@ -0,0 +1,3 @@
+go 1.18
+
+module test

go/ql/integration-tests/all-platforms/go/diagnostics/build-constraints-exclude-all-go-files/work/go.sum

@@ -0,0 +1,8 @@
+package test
+
+import "syscall/js"
+
+func Test() {
+	var x js.Error
+	_ = x
+}

go/ql/integration-tests/all-platforms/go/diagnostics/build-constraints-exclude-all-go-files/work/test.go

@@ -0,0 +1,14 @@
+{
+  "markdownMessage": "[Specify a custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages) that includes one or more `go build` commands to build the `.go` files to be analyzed.",
+  "severity": "error",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/go-files-found-but-not-processed",
+    "name": "Go files were found but not processed"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}