Merge pull request #224 from sauyon/no-vendor

Skip vendor directories for go.mod extraction

Author: smowton

change-notes/2020-10-01-gomod-extraction.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* The extractor now only extracts go.mod files belonging to extracted packages. In particular, vendored go.mod files will no longer be extracted unless the vendored package is explicitly passed to the extractor. This will remove unexpected `GoModExpr` and similar expressions seen by queries.

extractor/extractor.go

@@ -178,55 +178,32 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 			}
 
 			extractPackage(pkg, &wg, goroutineSem, fdSem)
-			return
-		}
 
-		log.Printf("Skipping dependency package %s.", pkg.PkgPath)
-	})
+			if pkgRoots[pkg.PkgPath] != "" {
+				modPath := filepath.Join(pkgRoots[pkg.PkgPath], "go.mod")
+				if util.FileExists(modPath) {
+					log.Printf("Extracting %s", modPath)
+					start := time.Now()
 
-	wg.Wait()
-
-	log.Println("Done extracting packages.")
-	log.Println("Starting to extract go.mod files.")
-
-	cwd, err := os.Getwd()
-	if err != nil {
-		log.Printf("Warning: unable to get working directory: %s", err.Error())
-		log.Println("Skipping go.mod extraction")
-	}
-	rcwd, err := filepath.EvalSymlinks(cwd)
-	if err == nil {
-		cwd = rcwd
-	}
-
-	goModPaths := make([]string, 0, 10)
+					err := extractGoMod(modPath)
+					if err != nil {
+						log.Printf("Failed to extract go.mod: %s", err.Error())
+					}
 
-	filepath.Walk(cwd, func(path string, info os.FileInfo, err error) error {
-		if filepath.Base(path) == "go.mod" && info != nil && info.Mode().IsRegular() {
-			if err != nil {
-				log.Printf("Found go.mod with path %s, but encountered error %s", path, err.Error())
+					end := time.Since(start)
+					log.Printf("Done extracting %s (%dms)", modPath, end.Nanoseconds()/1000000)
+				}
 			}
 
-			goModPaths = append(goModPaths, path)
+			return
 		}
 
-		return nil
+		log.Printf("Skipping dependency package %s.", pkg.PkgPath)
 	})
 
-	for _, path := range goModPaths {
-		log.Printf("Extracting %s", path)
-		start := time.Now()
-
-		err := extractGoMod(path)
-		if err != nil {
-			log.Printf("Failed to extract go.mod: %s", err.Error())
-		}
-
-		end := time.Since(start)
-		log.Printf("Done extracting %s (%dms)", path, end.Nanoseconds()/1000000)
-	}
+	wg.Wait()
 
-	log.Println("Done extracting go.mod files.")
+	log.Println("Done extracting packages.")
 
 	return nil
 }

ql/test/extractor-tests/go-mod-comments/stub.go

@@ -0,0 +1,3 @@
+package main
+
+func main() {}

ql/test/library-tests/semmle/go/GoModExpr/ExcludeLines.expected

@@ -0,0 +1,3 @@
+missingRequire
+missingExclude
+missingReplace

ql/test/library-tests/semmle/go/GoModExpr/ExcludeLines.ql

@@ -0,0 +1,88 @@
+import go
+
+/**
+ * Holds if there exists a comment on the same line as `l`
+ * that contains the substring "`kind`,`dep`,`ver`".
+ */
+predicate metadata(Locatable l, string kind, string mod, string dep, string ver) {
+  exists(string f, int line, Comment c, string text |
+    l.hasLocationInfo(f, line, _, _, _) and
+    c.hasLocationInfo(f, line, _, _, _)
+  |
+    text = c.getText().regexpFind("\\b([^,\\s]+,[^,]+,[^,]+,[^,\\s]+)", _, _) and
+    kind = text.regexpCapture("([^,]+),([^,]+),([^,]+),([^,]+)", 1) and
+    mod = text.regexpCapture("([^,]+),([^,]+),([^,]+),([^,]+)", 2) and
+    dep = text.regexpCapture("([^,]+),([^,]+),([^,]+),([^,]+)", 3) and
+    ver = text.regexpCapture("([^,]+),([^,]+),([^,]+),([^,]+)", 4)
+  )
+}
+
+query predicate missingRequire(string mod, string dep, string ver, int line) {
+  exists(Locatable l | metadata(l, "RequireLine", mod, dep, ver) |
+    l.hasLocationInfo(_, line, _, _, _)
+  ) and
+  not exists(GoModRequireLine req |
+    req.getModulePath() = mod and
+    req.getPath() = dep and
+    req.getVersion() = ver and
+    metadata(req, "RequireLine", mod, dep, ver) and
+    req.hasLocationInfo(_, line, _, _, _)
+  )
+}
+
+query predicate missingExclude(string mod, string dep, string ver, int line) {
+  exists(Locatable l | metadata(l, "ExcludeLine", mod, dep, ver) |
+    l.hasLocationInfo(_, line, _, _, _)
+  ) and
+  not exists(GoModExcludeLine exc |
+    exc.getModulePath() = mod and
+    exc.getPath() = dep and
+    exc.getVersion() = ver and
+    metadata(exc, "ExcludeLine", mod, dep, ver) and
+    exc.hasLocationInfo(_, line, _, _, _)
+  )
+}
+
+/**
+ * Holds if there exists a comment on the same line as `l`
+ * that contains the substring "ReplaceLine,`mod`,`dep`,`dver`,`rep`,`rver`".
+ */
+predicate repmetadata(Locatable l, string mod, string dep, string dver, string rep, string rver) {
+  exists(string f, int line, Comment c, string text |
+    l.hasLocationInfo(f, line, _, _, _) and
+    c.hasLocationInfo(f, line, _, _, _)
+  |
+    text = c.getText().regexpFind("\\b(ReplaceLine,[^,]*,[^,]*,[^,]*,[^,]*,[^,\\s]*)", _, _) and
+    mod = text.regexpCapture("ReplaceLine,([^,]*),([^,]*),([^,]*),([^,]*),([^,]*)", 1) and
+    dep = text.regexpCapture("ReplaceLine,([^,]*),([^,]*),([^,]*),([^,]*),([^,]*)", 2) and
+    dver = text.regexpCapture("ReplaceLine,([^,]*),([^,]*),([^,]*),([^,]*),([^,]*)", 3) and
+    rep = text.regexpCapture("ReplaceLine,([^,]*),([^,]*),([^,]*),([^,]*),([^,]*)", 4) and
+    rver = text.regexpCapture("ReplaceLine,([^,]*),([^,]*),([^,]*),([^,]*),([^,]*)", 5)
+  )
+}
+
+query predicate missingReplace(string mod, string dep, string dver, string rep, string rver, int line) {
+  exists(Locatable l | repmetadata(l, mod, dep, dver, rep, rver) |
+    l.hasLocationInfo(_, line, _, _, _)
+  ) and
+  not exists(GoModReplaceLine repl |
+    (
+      rver = repl.getReplacementVersion()
+      or
+      not exists(repl.getReplacementVersion()) and
+      rver = ""
+    ) and
+    (
+      dver = repl.getOriginalVersion()
+      or
+      not exists(repl.getOriginalVersion()) and
+      dver = ""
+    )
+  |
+    repl.getModulePath() = mod and
+    repl.getOriginalPath() = dep and
+    repl.getReplacementPath() = rep and
+    repmetadata(repl, mod, dep, dver, rep, rver) and
+    repl.hasLocationInfo(_, line, _, _, _)
+  )
+}