all: merge master (49d48a0) into gopls-release-branch.0.8

Also update gopls/go.mod with a replace directive.

For golang/go#51659

Merge List:

+ 2022-03-14 49d48a0 go/analysis/passes/composite: allow InternalFuzzTarget
+ 2022-03-11 198cae3 go/ssa: split pkg() into different cases for *Package and *types.Package
+ 2022-03-10 ee31f70 internal/lsp: add completion for use directives
+ 2022-03-10 622cf7b internal/lsp/cache: copy workFile when invalidating workspace
+ 2022-03-10 e7a12a3 go/ssa: add type substitution
+ 2022-03-10 c773560 internal/lsp/cache: set GOWORK=off when mutating modfiles
+ 2022-03-10 613589d internal/lsp: more precise completions for *testing.F fuzz methods
+ 2022-03-10 9f99e95 go/analysis: add analyzer for f.Add
+ 2022-03-10 7b442e3 go/callgraph/vta: fix package doc
+ 2022-03-09 1670aad go/analysis: fix bug with finding field type
+ 2022-03-09 fd72fd6 go/ssa/interp: adding external functions for tests
+ 2022-03-09 b105aac internal/lsp: use regexp to add go mod edit -go quick fix
+ 2022-03-09 03d333a internal/lsp: add snippet completion for function type parameters
+ 2022-03-09 94322c4 go/internal/gcimporter: pass -p to compiler in test
+ 2022-03-07 b59c441 internal/lsp/cache: always consider go.work files for ws expansion
+ 2022-03-04 e155b03 cmd/getgo: exec main from TestMain instead of running 'go build' in tests
+ 2022-03-04 e562276 internal/lsp: add hover for go.work use statements
+ 2022-03-04 121d1e4 internal/lsp: report diagnostics on go.work files
+ 2022-03-04 0eabed7 cmd/file2fuzz: exec main from TestMain instead of running 'go build' in tests
+ 2022-03-04 19fe2d7 gopls: update xurls dependency, remove \b workaround
+ 2022-03-03 3ce7728 internal/lsp/source: support stubbing concrete type params
+ 2022-03-03 3286927 internal/lsp/cache: construct workspace even when go.work has error
+ 2022-03-02 09e0201 go/ssa: determine whether a type is parameterized.
+ 2022-03-02 e43402d go/ssa: changes Package.values to objects.
+ 2022-03-02 a972457 go/ssa: adds *types.Info field to ssa.Function.
+ 2022-03-02 fc47946 go/ssa: Move BasicBlock operations into block.go
+ 2022-03-02 7103138 gopls: add regtest for edit go directive quick fix
+ 2022-03-02 4a5e7f0 internal/lsp: correctly apply file edits for edit go directive
+ 2022-03-02 6a6eb59 go/ssa: Put type canonicalization on own mutex.
+ 2022-03-02 afc5fce gopls/doc: address additional comments on workspace.md
+ 2022-03-02 abbbcaf gopls/doc: update the documentation for workspaces to suggest go.work
+ 2022-03-02 72442fe gopls: update neovim documentation for imitating goimports
+ 2022-03-01 ffa170d internal/jsonrpc2_v2: fix a racy map assignment in readIncoming
+ 2022-03-01 625c871 gopls: update neovim documentation for using go.work
+ 2022-03-01 fb3622a signature-generator: add Go func signature fuzzing tools
+ 2022-03-01 5d35a75 internal/jsonrpc2_v2: clarify documentation

Change-Id: Iff6ca42c407f6e63b3f99453d523dec4bcb44b1f

Author: findleyr

cmd/file2fuzz/main_test.go

@@ -5,67 +5,41 @@
 package main
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"runtime"
 	"strings"
 	"sync"
 	"testing"
 )
 
-// The setup for this test is mostly cribbed from x/exp/txtar.
+func TestMain(m *testing.M) {
+	if os.Getenv("GO_FILE2FUZZ_TEST_IS_FILE2FUZZ") != "" {
+		main()
+		os.Exit(0)
+	}
+
+	os.Exit(m.Run())
+}
 
-var buildBin struct {
+var f2f struct {
 	once sync.Once
-	name string
+	path string
 	err  error
 }
 
-func binPath(t *testing.T) string {
-	t.Helper()
-	if _, err := exec.LookPath("go"); err != nil {
-		t.Skipf("cannot build file2fuzz binary: %v", err)
-	}
-
-	buildBin.once.Do(func() {
-		exe, err := ioutil.TempFile("", "file2fuzz-*.exe")
-		if err != nil {
-			buildBin.err = err
-			return
-		}
-		exe.Close()
-		buildBin.name = exe.Name()
-
-		cmd := exec.Command("go", "build", "-o", buildBin.name, ".")
-		out, err := cmd.CombinedOutput()
-		if err != nil {
-			buildBin.err = fmt.Errorf("%s: %v\n%s", strings.Join(cmd.Args, " "), err, out)
-		}
+func file2fuzz(t *testing.T, dir string, args []string, stdin string) (string, bool) {
+	f2f.once.Do(func() {
+		f2f.path, f2f.err = os.Executable()
 	})
-
-	if buildBin.err != nil {
-		if runtime.GOOS == "android" {
-			t.Skipf("skipping test after failing to build file2fuzz binary: go_android_exec may have failed to copy needed dependencies (see https://golang.org/issue/37088)")
-		}
-		t.Fatal(buildBin.err)
-	}
-	return buildBin.name
-}
-
-func TestMain(m *testing.M) {
-	os.Exit(m.Run())
-	if buildBin.name != "" {
-		os.Remove(buildBin.name)
+	if f2f.err != nil {
+		t.Fatal(f2f.err)
 	}
-}
 
-func file2fuzz(t *testing.T, dir string, args []string, stdin string) (string, bool) {
-	t.Helper()
-	cmd := exec.Command(binPath(t), args...)
+	cmd := exec.Command(f2f.path, args...)
 	cmd.Dir = dir
+	cmd.Env = append(os.Environ(), "PWD="+dir, "GO_FILE2FUZZ_TEST_IS_FILE2FUZZ=1")
 	if stdin != "" {
 		cmd.Stdin = strings.NewReader(stdin)
 	}

cmd/getgo/.gitignore

@@ -1,3 +1,2 @@
 build
-testgetgo
 getgo

cmd/getgo/main_test.go

@@ -13,50 +13,27 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
-	"runtime"
 	"testing"
 )
 
-const (
-	testbin = "testgetgo"
-)
-
-var (
-	exeSuffix string // ".exe" on Windows
-)
-
-func init() {
-	if runtime.GOOS == "windows" {
-		exeSuffix = ".exe"
+func TestMain(m *testing.M) {
+	if os.Getenv("GO_GETGO_TEST_IS_GETGO") != "" {
+		main()
+		os.Exit(0)
 	}
-}
 
-// TestMain creates a getgo command for testing purposes and
-// deletes it after the tests have been run.
-func TestMain(m *testing.M) {
 	if os.Getenv("GOGET_INTEGRATION") == "" {
 		fmt.Fprintln(os.Stderr, "main_test: Skipping integration tests with GOGET_INTEGRATION unset")
 		return
 	}
 
-	args := []string{"build", "-tags", testbin, "-o", testbin + exeSuffix}
-	out, err := exec.Command("go", args...).CombinedOutput()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "building %s failed: %v\n%s", testbin, err, out)
-		os.Exit(2)
-	}
-
 	// Don't let these environment variables confuse the test.
 	os.Unsetenv("GOBIN")
 	os.Unsetenv("GOPATH")
 	os.Unsetenv("GIT_ALLOW_PROTOCOL")
 	os.Unsetenv("PATH")
 
-	r := m.Run()
-
-	os.Remove(testbin + exeSuffix)
-
-	os.Exit(r)
+	os.Exit(m.Run())
 }
 
 func createTmpHome(t *testing.T) string {
@@ -72,12 +49,18 @@ func createTmpHome(t *testing.T) string {
 // doRun runs the test getgo command, recording stdout and stderr and
 // returning exit status.
 func doRun(t *testing.T, args ...string) error {
+	exe, err := os.Executable()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Helper()
+
+	t.Logf("running getgo %v", args)
 	var stdout, stderr bytes.Buffer
-	t.Logf("running %s %v", testbin, args)
-	cmd := exec.Command("./"+testbin+exeSuffix, args...)
+	cmd := exec.Command(exe, args...)
 	cmd.Stdout = &stdout
 	cmd.Stderr = &stderr
-	cmd.Env = os.Environ()
+	cmd.Env = append(os.Environ(), "GO_GETGO_TEST_IS_GETGO=1")
 	status := cmd.Run()
 	if stdout.Len() > 0 {
 		t.Log("standard output:")

cmd/signature-fuzzer/README.md

@@ -0,0 +1,159 @@
+# signature-fuzzer
+
+This directory contains utilities for fuzz testing of Go function signatures, for use in developing/testing a Go compiler.
+
+The basic idea of the fuzzer is that it emits source code for a stand-alone Go program; this generated program is a series of pairs of functions, a "Caller" function and a "Checker" function. The signature of the Checker function is generated randomly (random number of parameters and returns, each with randomly chosen types). The "Caller" func contains invocations of the "Checker" function, each passing randomly chosen values to the params of the "Checker", then the caller verifies that expected values are returned correctly. The "Checker" function in turn has code to verify that the expected values (more details below).
+
+There are three main parts to the fuzzer: a generator package, a driver package, and a runner package.
+
+The "generator" contains the guts of the fuzzer, the bits that actually emit the random code.
+
+The "driver" is a stand-alone program that invokes the generator to create a single test program. It is not terribly useful on its own (since it doesn't actually build or run the generated program), but it is handy for debugging the generator or looking at examples of the emitted code.
+
+The "runner" is a more complete test harness; it repeatedly runs the generator to create a new test program, builds the test program, then runs it (checking for errors along the way). If at any point a build or test fails, the "runner" harness attempts a minimization process to try to narrow down the failure to a single package and/or function.
+
+## What the generated code looks like
+
+Generated Go functions will have an "interesting" set of signatures (mix of
+arrays, scalars, structs), intended to pick out corner cases and odd bits in the
+Go compiler's code that handles function calls and returns.
+
+The first generated file is genChecker.go, which contains function that look something
+like this (simplified):
+
+```
+type StructF4S0 struct {
+F0 float64
+F1 int16
+F2 uint16
+}
+
+// 0 returns 2 params
+func Test4(p0 int8, p1 StructF4S0)  {
+  c0 := int8(-1)
+  if p0 != c0 {
+    NoteFailure(4, "parm", 0)
+  }
+  c1 := StructF4S0{float64(2), int16(-3), uint16(4)}
+  if p1 != c1 {
+    NoteFailure(4, "parm", 1)
+  }
+  return
+}
+```
+
+Here the test generator has randomly selected 0 return values and 2 params, then randomly generated types for the params.
+
+The generator then emits code on the calling side into the file "genCaller.go", which might look like:
+
+```
+func Caller4() {
+var p0 int8
+p0 = int8(-1)
+var p1 StructF4S0
+p1 = StructF4S0{float64(2), int16(-3), uint16(4)}
+// 0 returns 2 params
+Test4(p0, p1)
+}
+```
+
+The generator then emits some utility functions (ex: NoteFailure) and a main routine that cycles through all of the tests.
+
+## Trying a single run of the generator
+
+To generate a set of source files just to see what they look like, you can build and run the test generator as follows. This creates a new directory "cabiTest" containing generated test files:
+
+```
+$ git clone https://golang.org/x/tools
+$ cd tools/cmd/signature-fuzzer/fuzz-driver
+$ go build .
+$ ./fuzz-driver -numpkgs 3 -numfcns 5 -seed 12345 -outdir /tmp/sigfuzzTest -pkgpath foobar
+$ cd /tmp/sigfuzzTest
+$ find . -type f -print
+./genCaller1/genCaller1.go
+./genUtils/genUtils.go
+./genChecker1/genChecker1.go
+./genChecker0/genChecker0.go
+./genCaller2/genCaller2.go
+./genCaller0/genCaller0.go
+./genMain.go
+./go.mod
+./genChecker2/genChecker2.go
+$
+```
+
+You can build and run the generated files in the usual way:
+
+```
+$ cd /tmp/sigfuzzTest
+$ go build .
+$ ./foobar
+starting main
+finished 15 tests
+$
+
+```
+
+## Example usage for the test runner
+
+The test runner orchestrates multiple runs of the fuzzer, iteratively emitting code, building it, and testing the resulting binary. To use the runner, build and invoke it with a specific number of iterations; it will select a new random seed on each invocation. The runner will terminate as soon as it finds a failure. Example:
+
+```
+$ git clone https://golang.org/x/tools
+$ cd tools/cmd/signature-fuzzer/fuzz-runner
+$ go build .
+$ ./fuzz-runner -numit=3
+... begin iteration 0 with current seed 67104558
+starting main
+finished 1000 tests
+... begin iteration 1 with current seed 67104659
+starting main
+finished 1000 tests
+... begin iteration 2 with current seed 67104760
+starting main
+finished 1000 tests
+$
+```
+
+If the runner encounters a failure, it will try to perform test-case "minimization", e.g. attempt to isolate the failure
+
+```
+$ cd tools/cmd/signature-fuzzer/fuzz-runner
+$ go build .
+$ ./fuzz-runner -n=10
+./fuzz-runner -n=10
+... begin iteration 0 with current seed 40661762
+Error: fail [reflect] |20|3|1| =Checker3.Test1= return 1
+error executing cmd ./fzTest: exit status 1
+... starting minimization for failed directory /tmp/fuzzrun1005327337/fuzzTest
+package minimization succeeded: found bad pkg 3
+function minimization succeeded: found bad fcn 1
+$
+```
+
+Here the runner has generates a failure, minimized it down to a single function and package, and left the resulting program in the output directory /tmp/fuzzrun1005327337/fuzzTest.
+
+## Limitations, future work
+
+No support yet for variadic functions.
+
+The set of generated types is still a bit thin; it has fairly limited support for interface values, and doesn't include channels.
+
+Todos:
+
+- better interface value coverage
+
+- implement testing of reflect.MakeFunc
+
+- extend to work with generic code of various types
+
+- extend to work in a debugging scenario (e.g. instead of just emitting code,
+  emit a script of debugger commands to run the program with expected
+  responses from the debugger)
+
+- rework things so that instead of always checking all of a given parameter
+  value, we sometimes skip over elements (or just check the length of a slice
+  or string as opposed to looking at its value)
+
+- consider adding runtime.GC() calls at some points in the generated code
+