Skip to content

Commit eaef544

Browse files
mbgmbg
authored
Merge pull request github#15810 from github/mbg/go/fix-initialised-module-names
2 parents 92e91f5 + 40ff75d commit eaef544

File tree

20 files changed

+247
-65
lines changed

20 files changed

+247
-65
lines changed

‎go/extractor/cli/go-autobuilder/go-autobuilder.go

Lines changed: 1 addition & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package main
33
import (
44
"fmt"
55
"log"
6-
"net/url"
76
"os"
87
"os/exec"
98
"path/filepath"
@@ -56,63 +55,6 @@ Build behavior:
5655
fmt.Fprintf(os.Stderr, "Usage:\n\n %s\n", os.Args[0])
5756
}
5857

59-
// Returns the import path of the package being built, or "" if it cannot be determined.
60-
func getImportPath() (importpath string) {
61-
importpath = os.Getenv("LGTM_INDEX_IMPORT_PATH")
62-
if importpath == "" {
63-
repourl := os.Getenv("SEMMLE_REPO_URL")
64-
if repourl == "" {
65-
githubrepo := os.Getenv("GITHUB_REPOSITORY")
66-
if githubrepo == "" {
67-
log.Printf("Unable to determine import path, as neither LGTM_INDEX_IMPORT_PATH nor GITHUB_REPOSITORY is set\n")
68-
return ""
69-
} else {
70-
importpath = "github.com/" + githubrepo
71-
}
72-
} else {
73-
importpath = getImportPathFromRepoURL(repourl)
74-
if importpath == "" {
75-
log.Printf("Failed to determine import path from SEMMLE_REPO_URL '%s'\n", repourl)
76-
return
77-
}
78-
}
79-
}
80-
log.Printf("Import path is '%s'\n", importpath)
81-
return
82-
}
83-
84-
// Returns the import path of the package being built from `repourl`, or "" if it cannot be
85-
// determined.
86-
func getImportPathFromRepoURL(repourl string) string {
87-
// check for scp-like URL as in "[email protected]:github/codeql-go.git"
88-
shorturl := regexp.MustCompile(`^([^@]+@)?([^:]+):([^/].*?)(\.git)?$`)
89-
m := shorturl.FindStringSubmatch(repourl)
90-
if m != nil {
91-
return m[2] + "/" + m[3]
92-
}
93-
94-
// otherwise parse as proper URL
95-
u, err := url.Parse(repourl)
96-
if err != nil {
97-
log.Fatalf("Malformed repository URL '%s'\n", repourl)
98-
}
99-
100-
if u.Scheme == "file" {
101-
// we can't determine import paths from file paths
102-
return ""
103-
}
104-
105-
if u.Hostname() == "" || u.Path == "" {
106-
return ""
107-
}
108-
109-
host := u.Hostname()
110-
path := u.Path
111-
// strip off leading slashes and trailing `.git` if present
112-
path = regexp.MustCompile(`^/+|\.git$`).ReplaceAllString(path, "")
113-
return host + "/" + path
114-
}
115-
11658
func restoreRepoLayout(fromDir string, dirEntries []string, scratchDirName string, toDir string) {
11759
for _, dirEntry := range dirEntries {
11860
if dirEntry != scratchDirName {
@@ -568,7 +510,7 @@ func installDependenciesAndBuild() {
568510
if len(workspaces) == 1 {
569511
workspace := workspaces[0]
570512

571-
importpath := getImportPath()
513+
importpath := util.GetImportPath()
572514
needGopath := getNeedGopath(workspace, importpath)
573515

574516
inLGTM := os.Getenv("LGTM_SRC") != "" || os.Getenv("LGTM_INDEX_NEED_GOPATH") != ""

‎go/extractor/project/project.go

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -439,8 +439,9 @@ func getBuildRoots(emitDiagnostics bool) (goWorkspaces []GoWorkspace, totalModul
439439
for _, component := range components {
440440
path = filepath.Join(path, component)
441441

442-
// Try to initialize a `go.mod` file automatically for the stray source files.
443-
if !slices.Contains(goModDirs, path) {
442+
// Try to initialize a `go.mod` file automatically for the stray source files if
443+
// doing so would not place it in a parent directory of an existing `go.mod` file.
444+
if !startsWithAnyOf(path, goModDirs) {
444445
goWorkspaces = append(goWorkspaces, GoWorkspace{
445446
BaseDir: path,
446447
DepMode: GoGetNoModules,
@@ -477,6 +478,16 @@ func getBuildRoots(emitDiagnostics bool) (goWorkspaces []GoWorkspace, totalModul
477478
return
478479
}
479480

481+
// Determines whether `str` starts with any of `prefixes`.
482+
func startsWithAnyOf(str string, prefixes []string) bool {
483+
for _, prefix := range prefixes {
484+
if relPath, err := filepath.Rel(str, prefix); err == nil && !strings.HasPrefix(relPath, "..") {
485+
return true
486+
}
487+
}
488+
return false
489+
}
490+
480491
// Finds Go workspaces in the current working directory.
481492
func GetWorkspaceInfo(emitDiagnostics bool) []GoWorkspace {
482493
bazelPaths := slices.Concat(

‎go/extractor/project/project_test.go

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package project
2+
3+
import (
4+
"path/filepath"
5+
"testing"
6+
)
7+
8+
func testStartsWithAnyOf(t *testing.T, path string, prefix string, expectation bool) {
9+
result := startsWithAnyOf(path, []string{prefix})
10+
if result != expectation {
11+
t.Errorf("Expected startsWithAnyOf(%s, %s) to be %t, but it is %t.", path, prefix, expectation, result)
12+
}
13+
}
14+
15+
func TestStartsWithAnyOf(t *testing.T) {
16+
testStartsWithAnyOf(t, ".", ".", true)
17+
testStartsWithAnyOf(t, ".", "dir", true)
18+
testStartsWithAnyOf(t, ".", filepath.Join("foo", "bar"), true)
19+
testStartsWithAnyOf(t, "dir", "dir", true)
20+
testStartsWithAnyOf(t, "foo", filepath.Join("foo", "bar"), true)
21+
testStartsWithAnyOf(t, filepath.Join("foo", "bar"), filepath.Join("foo", "bar"), true)
22+
testStartsWithAnyOf(t, filepath.Join("foo", "bar"), filepath.Join("foo", "bar", "baz"), true)
23+
24+
testStartsWithAnyOf(t, filepath.Join("foo", "bar"), "foo", false)
25+
testStartsWithAnyOf(t, filepath.Join("foo", "bar"), "bar", false)
26+
testStartsWithAnyOf(t, filepath.Join("foo", "bar"), filepath.Join("foo", "baz"), false)
27+
}

‎go/extractor/toolchain/toolchain.go

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@ import (
55
"log"
66
"os"
77
"os/exec"
8+
"path/filepath"
89
"strings"
910

11+
"github.com/github/codeql-go/extractor/util"
1012
"golang.org/x/mod/semver"
1113
)
1214

@@ -81,7 +83,20 @@ func TidyModule(path string) *exec.Cmd {
8183

8284
// Run `go mod init` in the directory given by `path`.
8385
func InitModule(path string) *exec.Cmd {
84-
modInit := exec.Command("go", "mod", "init", "codeql/auto-project")
86+
moduleName := "codeql/auto-project"
87+
88+
if importpath := util.GetImportPath(); importpath != "" {
89+
// This should be something like `github.com/user/repo`
90+
moduleName = importpath
91+
92+
// If we are not initialising the new module in the root directory of the workspace,
93+
// append the relative path to the module name.
94+
if relPath, err := filepath.Rel(".", path); err != nil && relPath != "." {
95+
moduleName = moduleName + "/" + relPath
96+
}
97+
}
98+
99+
modInit := exec.Command("go", "mod", "init", moduleName)
85100
modInit.Dir = path
86101
return modInit
87102
}

‎go/extractor/util/util.go

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@ import (
66
"io"
77
"io/fs"
88
"log"
9+
"net/url"
910
"os"
1011
"os/exec"
1112
"path/filepath"
13+
"regexp"
1214
"runtime"
1315
"slices"
1416
"strings"
@@ -350,3 +352,60 @@ func GetParentDirs(paths []string) []string {
350352
}
351353
return dirs
352354
}
355+
356+
// Returns the import path of the package being built, or "" if it cannot be determined.
357+
func GetImportPath() (importpath string) {
358+
importpath = os.Getenv("LGTM_INDEX_IMPORT_PATH")
359+
if importpath == "" {
360+
repourl := os.Getenv("SEMMLE_REPO_URL")
361+
if repourl == "" {
362+
githubrepo := os.Getenv("GITHUB_REPOSITORY")
363+
if githubrepo == "" {
364+
log.Printf("Unable to determine import path, as neither LGTM_INDEX_IMPORT_PATH nor GITHUB_REPOSITORY is set\n")
365+
return ""
366+
} else {
367+
importpath = "github.com/" + githubrepo
368+
}
369+
} else {
370+
importpath = getImportPathFromRepoURL(repourl)
371+
if importpath == "" {
372+
log.Printf("Failed to determine import path from SEMMLE_REPO_URL '%s'\n", repourl)
373+
return
374+
}
375+
}
376+
}
377+
log.Printf("Import path is '%s'\n", importpath)
378+
return
379+
}
380+
381+
// Returns the import path of the package being built from `repourl`, or "" if it cannot be
382+
// determined.
383+
func getImportPathFromRepoURL(repourl string) string {
384+
// check for scp-like URL as in "[email protected]:github/codeql-go.git"
385+
shorturl := regexp.MustCompile(`^([^@]+@)?([^:]+):([^/].*?)(\.git)?$`)
386+
m := shorturl.FindStringSubmatch(repourl)
387+
if m != nil {
388+
return m[2] + "/" + m[3]
389+
}
390+
391+
// otherwise parse as proper URL
392+
u, err := url.Parse(repourl)
393+
if err != nil {
394+
log.Fatalf("Malformed repository URL '%s'\n", repourl)
395+
}
396+
397+
if u.Scheme == "file" {
398+
// we can't determine import paths from file paths
399+
return ""
400+
}
401+
402+
if u.Hostname() == "" || u.Path == "" {
403+
return ""
404+
}
405+
406+
host := u.Hostname()
407+
path := u.Path
408+
// strip off leading slashes and trailing `.git` if present
409+
path = regexp.MustCompile(`^/+|\.git$`).ReplaceAllString(path, "")
410+
return host + "/" + path
411+
}

‎go/extractor/cli/go-autobuilder/go-autobuilder_test.go renamed to ‎go/extractor/util/util_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package main
1+
package util
22

33
import "testing"
44

‎go/ql/integration-tests/all-platforms/go/mixed-layout/diagnostics.expected

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{
2+
"markdownMessage": "1 `go.work` file was found:\n\n`workspace/go.work`",
3+
"severity": "note",
4+
"source": {
5+
"extractorName": "go",
6+
"id": "go/autobuilder/go-work-found",
7+
"name": "`go.work` file found"
8+
},
9+
"visibility": {
10+
"cliSummaryTable": false,
11+
"statusPage": false,
12+
"telemetry": true
13+
}
14+
}
15+
{
16+
"markdownMessage": "Go files were found outside of the Go modules corresponding to these `go.mod` files.\n\n`workspace/subdir/go.mod`, `module/go.mod`",
17+
"severity": "note",
18+
"source": {
19+
"extractorName": "go",
20+
"id": "go/autobuilder/go-files-outside-go-modules",
21+
"name": "Go files were found outside Go modules"
22+
},
23+
"visibility": {
24+
"cliSummaryTable": false,
25+
"statusPage": false,
26+
"telemetry": true
27+
}
28+
}

‎go/ql/integration-tests/all-platforms/go/mixed-layout/force_sequential_test_execution

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# go get has been observed to sometimes fail when multiple tests try to simultaneously fetch the same package.
2+
goget

‎go/ql/integration-tests/all-platforms/go/mixed-layout/src/module/go.mod

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
go 1.14
2+
3+
require golang.org/x/net v0.0.0-20200505041828-1ed23360d12c
4+
5+
module module

‎go/ql/integration-tests/all-platforms/go/mixed-layout/src/module/go.sum

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
2+
golang.org/x/net v0.0.0-20200505041828-1ed23360d12c h1:zJ0mtu4jCalhKg6Oaukv6iIkb+cOvDrajDH9DH46Q4M=
3+
golang.org/x/net v0.0.0-20200505041828-1ed23360d12c/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
4+
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
5+
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
6+
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
7+
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

0 commit comments

Comments
 (0)