Merge pull request github#13589 from owen-mc/go/tolerate-go-mod-not-in-project-root

Go: Deal better with a single go.mod file which is not in the project root

Author: owen-mc

go/extractor/cli/go-autobuilder/go-autobuilder.go

@@ -129,16 +129,19 @@ func EmitPackageDifferentOSArchitecture(pkgPath string) {
 	)
 }
 
+func plural(n int, singular, plural string) string {
+	if n == 1 {
+		return singular
+	} else {
+		return plural
+	}
+}
+
 const maxNumPkgPaths = 5
 
 func EmitCannotFindPackages(pkgPaths []string) {
 	numPkgPaths := len(pkgPaths)
 
-	ending := "s"
-	if numPkgPaths == 1 {
-		ending = ""
-	}
-
 	numPrinted := numPkgPaths
 	truncated := false
 	if numPrinted > maxNumPkgPaths {
@@ -154,7 +157,11 @@ func EmitCannotFindPackages(pkgPaths []string) {
 	emitDiagnostic(
 		"go/autobuilder/package-not-found",
 		"Some packages could not be found",
-		fmt.Sprintf("%d package%s could not be found:\n\n%s.\n\nDefinitions in those packages may not be recognized by CodeQL, and files that use them may only be partially analyzed.\n\nCheck that the paths are correct and make sure any private packages can be accessed. If any of the packages are present in the repository then you may need a [custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).", numPkgPaths, ending, secondLine),
+		fmt.Sprintf(
+			"%d package%s could not be found:\n\n%s.\n\nDefinitions in those packages may not be recognized by CodeQL, and files that use them may only be partially analyzed.\n\nCheck that the paths are correct and make sure any private packages can be accessed. If any of the packages are present in the repository then you may need a [custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
+			numPkgPaths,
+			plural(len(pkgPaths), "", "s"),
+			secondLine),
 		severityWarning,
 		fullVisibility,
 		noLocation,
@@ -194,6 +201,123 @@ func EmitRelativeImportPaths() {
 	)
 }
 
+// The following diagnostics are telemetry-only.
+
+func EmitBazelBuildFilesFound(bazelPaths []string) {
+	emitDiagnostic(
+		"go/autobuilder/bazel-build-file-found",
+		"Bazel BUILD files were found",
+		fmt.Sprintf(
+			"%d bazel BUILD %s found:\n\n`%s`",
+			len(bazelPaths),
+			plural(len(bazelPaths), "file was", "files were"),
+			strings.Join(bazelPaths, "`, `")),
+		severityNote,
+		telemetryOnly,
+		noLocation,
+	)
+}
+
+func EmitGopkgTomlFound() {
+	emitDiagnostic(
+		"go/autobuilder/gopkg-toml-found",
+		"A dep `Gopkg.toml` file was found",
+		"A dep `Gopkg.toml` file was found",
+		severityNote,
+		telemetryOnly,
+		noLocation,
+	)
+}
+
+func EmitGlideYamlFound() {
+	emitDiagnostic(
+		"go/autobuilder/glide-yaml-found",
+		"A Glide `glide.yaml` file was found",
+		"A Glide `glide.yaml` file was found",
+		severityNote,
+		telemetryOnly,
+		noLocation,
+	)
+}
+
+func EmitGoWorkFound(goWorkPaths []string) {
+	emitDiagnostic(
+		"go/autobuilder/go-work-found",
+		"`go.work` file found",
+		fmt.Sprintf(
+			"%d `go.work` %s found:\n\n`%s`",
+			len(goWorkPaths),
+			plural(len(goWorkPaths), "file was", "files were"),
+			strings.Join(goWorkPaths, "`, `")),
+		severityNote,
+		telemetryOnly,
+		noLocation,
+	)
+}
+
+func EmitGoFilesOutsideGoModules(goModPaths []string) {
+	emitDiagnostic(
+		"go/autobuilder/go-files-outside-go-modules",
+		"Go files were found outside Go modules",
+		"Go files were found outside of the Go modules corresponding to these `go.mod` files.\n\n`"+strings.Join(goModPaths, "`, `")+"`",
+		severityNote,
+		telemetryOnly,
+		noLocation,
+	)
+}
+
+func EmitMultipleGoModFoundNested(goModPaths []string) {
+	emitDiagnostic(
+		"go/autobuilder/multiple-go-mod-found-nested",
+		"Multiple `go.mod` files were found, all nested under one root `go.mod` file",
+		fmt.Sprintf(
+			"%d `go.mod` files were found:\n\n`%s`",
+			len(goModPaths),
+			strings.Join(goModPaths, "`, `")),
+		severityNote,
+		telemetryOnly,
+		noLocation,
+	)
+}
+
+func EmitMultipleGoModFoundNotNested(goModPaths []string) {
+	emitDiagnostic(
+		"go/autobuilder/multiple-go-mod-found-not-nested",
+		"Multiple `go.mod` files found, not all nested under one root `go.mod` file",
+		fmt.Sprintf(
+			"%d `go.mod` files were found:\n\n`%s`",
+			len(goModPaths),
+			strings.Join(goModPaths, "`, `")),
+		severityNote,
+		telemetryOnly,
+		noLocation,
+	)
+}
+
+func EmitSingleRootGoModFound(goModPath string) {
+	emitDiagnostic(
+		"go/autobuilder/single-root-go-mod-found",
+		"A single `go.mod` file was found in the root",
+		"A single `go.mod` file was found.\n\n`"+goModPath+"`",
+		severityNote,
+		telemetryOnly,
+		noLocation,
+	)
+}
+
+func EmitSingleNonRootGoModFound(goModPath string) {
+	emitDiagnostic(
+		"go/autobuilder/single-non-root-go-mod-found",
+		"A single, non-root `go.mod` file was found",
+		"A single, non-root `go.mod` file was found.\n\n`"+goModPath+"`",
+		severityNote,
+		telemetryOnly,
+		noLocation,
+	)
+}
+
+// The following diagnostics are related to identifying the build environment.
+
 func EmitNoGoModAndNoGoEnv(msg string) {
 	emitDiagnostic(
 		"go/autobuilder/env-no-go-mod-no-go-env",

go/extractor/diagnostics/diagnostics.go

@@ -297,3 +297,46 @@ func FindGoFiles(root string) bool {
 	})
 	return found
 }
+
+func FindAllFilesWithName(root string, name string, dirsToSkip ...string) []string {
+	paths := make([]string, 0, 1)
+	filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			for _, dirToSkip := range dirsToSkip {
+				if path == dirToSkip {
+					return filepath.SkipDir
+				}
+			}
+		}
+		if d.Name() == name {
+			paths = append(paths, path)
+		}
+		return nil
+	})
+	return paths
+}
+
+func AnyGoFilesOutsideDirs(root string, dirsToSkip ...string) bool {
+	found := false
+	filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			for _, dirToSkip := range dirsToSkip {
+				if path == dirToSkip {
+					return filepath.SkipDir
+				}
+			}
+		}
+		if filepath.Ext(d.Name()) == ".go" {
+			found = true
+			return filepath.SkipAll
+		}
+		return nil
+	})
+	return found
+}

go/extractor/util/util.go

@@ -0,0 +1,28 @@
+{
+  "markdownMessage": "1 bazel BUILD file was found:\n\n`BUILD.bazel`",
+  "severity": "note",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/bazel-build-file-found",
+    "name": "Bazel BUILD files were found"
+  },
+  "visibility": {
+    "cliSummaryTable": false,
+    "statusPage": false,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "A single `go.mod` file was found.\n\n`go.mod`",
+  "severity": "note",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/single-root-go-mod-found",
+    "name": "A single `go.mod` file was found in the root"
+  },
+  "visibility": {
+    "cliSummaryTable": false,
+    "statusPage": false,
+    "telemetry": true
+  }
+}

go/ql/integration-tests/all-platforms/go/bazel-sample-1/BUILD.bazel

@@ -0,0 +1,7 @@
+go 1.14
+
+require (
+	golang.org/x/net v0.0.0-20200505041828-1ed23360d12c
+)
+
+module bazelsample

go/ql/integration-tests/all-platforms/go/bazel-sample-1/diagnostics.expected

@@ -0,0 +1,7 @@
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20200505041828-1ed23360d12c h1:zJ0mtu4jCalhKg6Oaukv6iIkb+cOvDrajDH9DH46Q4M=
+golang.org/x/net v0.0.0-20200505041828-1ed23360d12c/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

go/ql/integration-tests/all-platforms/go/bazel-sample-1/go.mod

@@ -0,0 +1,7 @@
+htmlFiles
+extractionErrors
+| Extraction failed in test.go with error \tother declaration of test | 2 |
+| Extraction failed in todel.go with error test redeclared in this block | 2 |
+#select
+| test.go:0:0:0:0 | test.go |
+| todel.go:0:0:0:0 | todel.go |

go/ql/integration-tests/all-platforms/go/bazel-sample-1/go.sum

@@ -0,0 +1,12 @@
+package bazelsample
+
+import (
+	"golang.org/x/net/ipv4"
+)
+
+func test() {
+
+	header := ipv4.Header{}
+	header.Version = 4
+
+}

go/ql/integration-tests/all-platforms/go/bazel-sample-1/test.expected

@@ -0,0 +1,8 @@
+import sys
+
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+run_codeql_database_create([], lang="go")
+
+check_diagnostics()