[gopls-release-branch.0.5] all: merge master into gopls-release-branch.0.5

a9763ab internal/lsp: refactor code action go command calls
5bad459 internal/lsp: fix godef for embedded type aliases
45586dd internal/lsp: nest the workspace root if there is only one module
780cb80 internal/lsp: fix go.mod parse error parsing to show diagnostics
20be4ac internal/lsp: remove workspace packages as needed
6d15148 internal/lsp/cache: only invalidate metadata for go.mod files on save
a738094 internal/lsp: pass a parsed module to go mod tidy
c0d5e89 internal/lsp/fake: hold the mutex for all of SaveBufferWithoutActions
be796f8 gopls/internal/regtest: handle flake in TestCRLF
0557df3 internal/lsp/source: set experimentalPackageCacheKey to true by default
4783bc9 Handle BareInits option in ssa.BuilderMode
25dc3e1 internal/lsp: handle deletion of a directory on disk
4d9c4f8 internal/lsp/mod: speed up and simplify go.mod code lenses
1727ee1 internal/lsp/cache: fix a staticcheck warning by handling error
6ed8ff9 internal/lsp: support unimported completions in multi-module mode
598b068 internal/lsp: fix end positions for multi-line comments with CRLF lines
bd56c0a internal/lsp: use a marker test for struct field ranking test
74a3905 internal/lsp: order struct fields completions by definition
f544f6c gopls/internal/regtest: eliminate log duplication
9036a0f Revert "internal/lsp: explicitly watch all known directories"
3c3a812 internal/lsp: explicitly watch all known directories
74f2986 internal/lsp: show critical error pop-ups as progress reports
1d69943 go/packages: handle variation an an error message
6f6c72a internal/lsp: fix capability registration for semantic tokens
d68bbb5 internal/gocommand: improve debug string
247bdb2 go/packages: start with empty environment
1a2739c internal/lsp: get debug pages limping along again
ac45abd gopls/internal/regtest: update TestEditFile to use a modified buffer
9712d02 internal/lsp: add ?utm_source=gopls to links to pkg.go.dev
61ea331 internal/lsp/source: handle nil pointer exception in completion
1643af1 internal/regtest: refactor diagnostic expectation implementation
559c4ac internal/lsp: make log message for debug server clickable
eeaa07d internal/lsp: remove gopls-<name> configuration
b389559 internal/lsp/cache: fix InDir error comment
41a3a58 internal/lsp: fix recursive go generate pattern
5794f8b internal/lsp: trim progress report messages
69daaf9 internal/lsp: do not treat failed go test commands as errors
7099162 internal/lsp: remove dead code
092357f internal/lsp/cache: disable GOPACKAGESDRIVER
f6c1dd6 internal/lsp/cache: suppress Load log spam
53e0aa8 gopls/internal/regtest: add an option to nest the workdir
079ba7b internal/typesinternal: add a copy of the Go 1.16 go/types error codes
169ad6d internal/lsp: avoid diagnosing unopened non-workspace packages

Change-Id: If0097a1f85b583a1dc7400924df43920b9cf42f9

Author: findleyr

go/internal/packagesdriver/sizes.go

@@ -6,80 +6,16 @@
 package packagesdriver
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
 	"go/types"
-	"os/exec"
 	"strings"
 
 	"golang.org/x/tools/internal/gocommand"
 )
 
 var debug = false
 
-func GetSizes(ctx context.Context, buildFlags, env []string, gocmdRunner *gocommand.Runner, dir string) (types.Sizes, error) {
-	// TODO(matloob): Clean this up. This code is mostly a copy of packages.findExternalDriver.
-	const toolPrefix = "GOPACKAGESDRIVER="
-	tool := ""
-	for _, env := range env {
-		if val := strings.TrimPrefix(env, toolPrefix); val != env {
-			tool = val
-		}
-	}
-
-	if tool == "" {
-		var err error
-		tool, err = exec.LookPath("gopackagesdriver")
-		if err != nil {
-			// We did not find the driver, so use "go list".
-			tool = "off"
-		}
-	}
-
-	if tool == "off" {
-		inv := gocommand.Invocation{
-			BuildFlags: buildFlags,
-			Env:        env,
-			WorkingDir: dir,
-		}
-		return GetSizesGolist(ctx, inv, gocmdRunner)
-	}
-
-	req, err := json.Marshal(struct {
-		Command    string   `json:"command"`
-		Env        []string `json:"env"`
-		BuildFlags []string `json:"build_flags"`
-	}{
-		Command:    "sizes",
-		Env:        env,
-		BuildFlags: buildFlags,
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to encode message to driver tool: %v", err)
-	}
-
-	buf := new(bytes.Buffer)
-	cmd := exec.CommandContext(ctx, tool)
-	cmd.Dir = dir
-	cmd.Env = env
-	cmd.Stdin = bytes.NewReader(req)
-	cmd.Stdout = buf
-	cmd.Stderr = new(bytes.Buffer)
-	if err := cmd.Run(); err != nil {
-		return nil, fmt.Errorf("%v: %v: %s", tool, err, cmd.Stderr)
-	}
-	var response struct {
-		// Sizes, if not nil, is the types.Sizes to use when type checking.
-		Sizes *types.StdSizes
-	}
-	if err := json.Unmarshal(buf.Bytes(), &response); err != nil {
-		return nil, err
-	}
-	return response.Sizes, nil
-}
-
 func GetSizesGolist(ctx context.Context, inv gocommand.Invocation, gocmdRunner *gocommand.Runner) (types.Sizes, error) {
 	inv.Verb = "list"
 	inv.Args = []string{"-f", "{{context.GOARCH}} {{context.Compiler}}", "--", "unsafe"}

go/packages/external.go

@@ -89,7 +89,7 @@ func findExternalDriver(cfg *Config) driver {
 			return nil, fmt.Errorf("%v: %v: %s", tool, err, cmd.Stderr)
 		}
 		if len(stderr.Bytes()) != 0 && os.Getenv("GOPACKAGESPRINTDRIVERERRORS") != "" {
-			fmt.Fprintf(os.Stderr, "%s stderr: <<%s>>\n", cmdDebugStr(cmd, words...), stderr)
+			fmt.Fprintf(os.Stderr, "%s stderr: <<%s>>\n", cmdDebugStr(cmd), stderr)
 		}
 
 		var response driverResponse

go/packages/golist.go

@@ -827,6 +827,7 @@ func (state *golistState) cfgInvocation() gocommand.Invocation {
 		BuildFlags: cfg.BuildFlags,
 		ModFile:    cfg.modFile,
 		ModFlag:    cfg.modFlag,
+		CleanEnv:   cfg.Env != nil,
 		Env:        cfg.Env,
 		Logf:       cfg.Logf,
 		WorkingDir: cfg.Dir,
@@ -901,8 +902,13 @@ func (state *golistState) invokeGo(verb string, args ...string) (*bytes.Buffer,
 			return unicode.IsOneOf([]*unicode.RangeTable{unicode.L, unicode.M, unicode.N, unicode.P, unicode.S}, r) &&
 				!strings.ContainsRune("!\"#$%&'()*,:;<=>?[\\]^`{|}\uFFFD", r)
 		}
+		// golang/go#36770: Handle case where cmd/go prints module download messages before the error.
+		msg := stderr.String()
+		for strings.HasPrefix(msg, "go: downloading") {
+			msg = msg[strings.IndexRune(msg, '\n')+1:]
+		}
 		if len(stderr.String()) > 0 && strings.HasPrefix(stderr.String(), "# ") {
-			msg := stderr.String()[len("# "):]
+			msg := msg[len("# "):]
 			if strings.HasPrefix(strings.TrimLeftFunc(msg, isPkgPathRune), "\n") {
 				return stdout, nil
 			}
@@ -1069,17 +1075,22 @@ func containsGoFile(s []string) bool {
 	return false
 }
 
-func cmdDebugStr(cmd *exec.Cmd, args ...string) string {
+func cmdDebugStr(cmd *exec.Cmd) string {
 	env := make(map[string]string)
 	for _, kv := range cmd.Env {
-		split := strings.Split(kv, "=")
+		split := strings.SplitN(kv, "=", 2)
 		k, v := split[0], split[1]
 		env[k] = v
 	}
-	var quotedArgs []string
-	for _, arg := range args {
-		quotedArgs = append(quotedArgs, strconv.Quote(arg))
-	}
 
-	return fmt.Sprintf("GOROOT=%v GOPATH=%v GO111MODULE=%v PWD=%v go %s", env["GOROOT"], env["GOPATH"], env["GO111MODULE"], env["PWD"], strings.Join(quotedArgs, " "))
+	var args []string
+	for _, arg := range cmd.Args {
+		quoted := strconv.Quote(arg)
+		if quoted[1:len(quoted)-1] != arg || strings.Contains(arg, " ") {
+			args = append(args, quoted)
+		} else {
+			args = append(args, arg)
+		}
+	}
+	return fmt.Sprintf("GOROOT=%v GOPATH=%v GO111MODULE=%v GOPROXY=%v PWD=%v %v", env["GOROOT"], env["GOPATH"], env["GO111MODULE"], env["GOPROXY"], env["PWD"], strings.Join(args, " "))
 }

go/packages/packages_test.go

@@ -86,8 +86,8 @@ func TestLoadZeroConfig(t *testing.T) {
 	hash := initial[0]
 	// Even though the hash package has imports,
 	// they are not reported.
-	got := fmt.Sprintf("iamashamedtousethedisabledqueryname=%s srcs=%v imports=%v", hash.Name, srcs(hash), hash.Imports)
-	want := "iamashamedtousethedisabledqueryname=hash srcs=[hash.go] imports=map[]"
+	got := fmt.Sprintf("srcs=%v imports=%v", srcs(hash), hash.Imports)
+	want := "srcs=[hash.go] imports=map[]"
 	if got != want {
 		t.Fatalf("got %s, want %s", got, want)
 	}
@@ -2628,6 +2628,16 @@ func main() {
 	}
 }
 
+func TestEmptyEnvironment(t *testing.T) {
+	cfg := &packages.Config{
+		Env: []string{"FOO=BAR"},
+	}
+	_, err := packages.Load(cfg, "fmt")
+	if err == nil {
+		t.Fatal("Load with explicitly empty environment should fail")
+	}
+}
+
 func errorMessages(errors []packages.Error) []string {
 	var msgs []string
 	for _, err := range errors {

go/ssa/mode.go

@@ -66,6 +66,9 @@ func (m BuilderMode) String() string {
 	if m&BuildSerially != 0 {
 		buf.WriteByte('L')
 	}
+	if m&BareInits != 0 {
+		buf.WriteByte('I')
+	}
 	return buf.String()
 }
 
@@ -88,6 +91,8 @@ func (m *BuilderMode) Set(s string) error {
 			mode |= NaiveForm
 		case 'L':
 			mode |= BuildSerially
+		case 'I':
+			mode |= BareInits
 		default:
 			return fmt.Errorf("unknown BuilderMode option: %q", c)
 		}

gopls/doc/settings.md

@@ -249,9 +249,12 @@ semantic tokens to the client.
 
 Default: `false`.
 ### **expandWorkspaceToModule** *bool*
-expandWorkspaceToModule instructs `gopls` to expand the scope of the workspace to include the
-modules containing the workspace folders. Set this to false to avoid loading
-your entire module. This is particularly useful for those working in a monorepo.
+expandWorkspaceToModule instructs `gopls` to adjust the scope of the
+workspace to find the best available module root. `gopls` first looks for
+a go.mod file in any parent directory of the workspace folder, expanding
+the scope to that directory if it exists. If no viable parent directory is
+found, gopls will check if there is exactly one child directory containing
+a go.mod file, narrowing the scope to that directory if it exists.
 
 
 Default: `true`.
@@ -281,7 +284,7 @@ by an experiment because caching behavior is subtle and difficult to
 comprehensively test.
 
 
-Default: `false`.
+Default: `true`.
 <!-- END Experimental: DO NOT MANUALLY EDIT THIS SECTION -->
 
 ## Debugging

gopls/internal/regtest/diagnostics_test.go

@@ -852,28 +852,29 @@ func TestHello(t *testing.T) {
 
 // Reproduce golang/go#40690.
 func TestCreateOnlyXTest(t *testing.T) {
-	t.Skip("golang/go#40690 is not resolved yet.")
-
 	const mod = `
-	-- go.mod --
-	module mod.com
-	-- foo/foo.go --
-	package foo
-	-- foo/bar_test.go --
-	`
+-- go.mod --
+module mod.com
+-- foo/foo.go --
+package foo
+-- foo/bar_test.go --
+`
 	run(t, mod, func(t *testing.T, env *Env) {
 		env.OpenFile("foo/bar_test.go")
-		env.EditBuffer("foo/bar_test.go", fake.NewEdit(0, 0, 0, 0, `package foo
-	`))
+		env.EditBuffer("foo/bar_test.go", fake.NewEdit(0, 0, 0, 0, "package foo"))
 		env.Await(
 			CompletedWork(lsp.DiagnosticWorkTitle(lsp.FromDidChange), 1),
 		)
-		env.RegexpReplace("foo/bar_test.go", "package foo", "package foo_test")
+		env.RegexpReplace("foo/bar_test.go", "package foo", `package foo_test
+
+import "testing"
+
+func TestX(t *testing.T) {
+	var x int
+}
+`)
 		env.Await(
-			OnceMet(
-				CompletedWork(lsp.DiagnosticWorkTitle(lsp.FromDidChange), 2),
-				NoErrorLogs(),
-			),
+			env.DiagnosticAtRegexp("foo/bar_test.go", "x"),
 		)
 	})
 }
@@ -1246,7 +1247,7 @@ func main() {
 			env.DiagnosticAtRegexp("main.go", "x"),
 		)
 	})
-	withOptions(WithRootPath("a"), WithLimitWorkspaceScope()).run(t, mod, func(t *testing.T, env *Env) {
+	withOptions(WithRootPath("a"), LimitWorkspaceScope()).run(t, mod, func(t *testing.T, env *Env) {
 		env.OpenFile("a/main.go")
 		env.Await(
 			NoDiagnostics("main.go"),
@@ -1422,7 +1423,7 @@ func main() {
 		env.Await(
 			OnceMet(
 				InitialWorkspaceLoad,
-				NoDiagnosticWithMessage("illegal character U+0023 '#'"),
+				NoDiagnosticWithMessage("", "illegal character U+0023 '#'"),
 			),
 		)
 	})
@@ -1452,3 +1453,65 @@ package foo_`
 		)
 	})
 }
+
+// TestProgressBarErrors confirms that critical workspace load errors are shown
+// and updated via progress reports.
+func TestProgressBarErrors(t *testing.T) {
+	testenv.NeedsGo1Point(t, 14)
+
+	const pkg = `
+-- go.mod --
+modul mod.com
+
+go 1.12
+-- main.go --
+package main
+`
+	run(t, pkg, func(t *testing.T, env *Env) {
+		env.OpenFile("go.mod")
+		env.Await(
+			OutstandingWork("Error loading workspace", "unknown directive"),
+		)
+		env.EditBuffer("go.mod", fake.NewEdit(0, 0, 3, 0, `module mod.com
+
+go 1.hello
+`))
+		env.Await(
+			OutstandingWork("Error loading workspace", "invalid go version"),
+		)
+		env.RegexpReplace("go.mod", "go 1.hello", "go 1.12")
+		env.Await(
+			NoOutstandingWork(),
+		)
+	})
+}
+
+func TestDeleteDirectory(t *testing.T) {
+	testenv.NeedsGo1Point(t, 14)
+
+	const mod = `
+-- bob/bob.go --
+package bob
+
+func Hello() {
+	var x int
+}
+-- go.mod --
+module mod.com
+-- main.go --
+package main
+
+import "mod.com/bob"
+
+func main() {
+	bob.Hello()
+}
+`
+	run(t, mod, func(t *testing.T, env *Env) {
+		env.RemoveWorkspaceFile("bob")
+		env.Await(
+			env.DiagnosticAtRegexp("main.go", `"mod.com/bob"`),
+			EmptyDiagnostics("bob/bob.go"),
+		)
+	})
+}

gopls/internal/regtest/env.go

@@ -58,8 +58,8 @@ type State struct {
 }
 
 type workProgress struct {
-	title   string
-	percent float64
+	title, msg string
+	percent    float64
 }
 
 func (s State) String() string {
@@ -200,10 +200,16 @@ func (e *Env) onProgress(_ context.Context, m *protocol.ProgressParams) error {
 	switch kind := v["kind"]; kind {
 	case "begin":
 		work.title = v["title"].(string)
+		if msg, ok := v["message"]; ok {
+			work.msg = msg.(string)
+		}
 	case "report":
 		if pct, ok := v["percentage"]; ok {
 			work.percent = pct.(float64)
 		}
+		if msg, ok := v["message"]; ok {
+			work.msg = msg.(string)
+		}
 	case "end":
 		title := e.state.outstandingWork[m.Token].title
 		e.state.completedWork[title] = e.state.completedWork[title] + 1