gopls/internal/lsp/cmd: add a stats -anon flag to show anonymous data

Also alter Bug storage to serialize bugs as JSON.

Change-Id: I58ef96181b6c233333d1dfff39f1587f3cc9dd35
Reviewed-on: https://go-review.googlesource.com/c/tools/+/497755
Run-TryBot: Robert Findley <[email protected]>
gopls-CI: kokoro <[email protected]>
Reviewed-by: Alan Donovan <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>

Author: findleyr

gopls/internal/bug/bug.go

@@ -17,6 +17,7 @@ import (
 	"runtime/debug"
 	"sort"
 	"sync"
+	"time"
 )
 
 // PanicOnBugs controls whether to panic when bugs are reported.
@@ -32,12 +33,15 @@ var (
 
 // A Bug represents an unexpected event or broken invariant. They are used for
 // capturing metadata that helps us understand the event.
+//
+// Bugs are JSON-serializable.
 type Bug struct {
-	File        string // file containing the call to bug.Report
-	Line        int    // line containing the call to bug.Report
-	Description string // description of the bug
-	Key         string // key identifying the bug (file:line if available)
-	Stack       string // call stack
+	File        string    // file containing the call to bug.Report
+	Line        int       // line containing the call to bug.Report
+	Description string    // description of the bug
+	Key         string    // key identifying the bug (file:line if available)
+	Stack       string    // call stack
+	AtTime      time.Time // time the bug was reported
 }
 
 // Reportf reports a formatted bug message.
@@ -77,6 +81,7 @@ func report(description string) {
 		Description: description,
 		Key:         key,
 		Stack:       string(debug.Stack()),
+		AtTime:      time.Now(),
 	}
 
 	mu.Lock()

gopls/internal/bug/bug_test.go

@@ -5,8 +5,12 @@
 package bug
 
 import (
+	"encoding/json"
 	"fmt"
 	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
 )
 
 func resetForTesting() {
@@ -62,3 +66,26 @@ func TestBugHandler(t *testing.T) {
 		t.Errorf("got %q, want %q", got, want)
 	}
 }
+
+func TestBugJSON(t *testing.T) {
+	b1 := Bug{
+		File:        "foo.go",
+		Line:        1,
+		Description: "a bug",
+		Key:         "foo.go:1",
+		Stack:       "<stack>",
+		AtTime:      time.Now(),
+	}
+
+	data, err := json.Marshal(b1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var b2 Bug
+	if err := json.Unmarshal(data, &b2); err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(b1, b2); diff != "" {
+		t.Errorf("bugs differ after JSON Marshal/Unmarshal (-b1 +b2):\n%s", diff)
+	}
+}

gopls/internal/lsp/cmd/stats.go

@@ -9,9 +9,11 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"go/token"
 	"io/fs"
 	"os"
 	"path/filepath"
+	"reflect"
 	"runtime"
 	"strings"
 	"sync"
@@ -24,10 +26,13 @@ import (
 	"golang.org/x/tools/gopls/internal/lsp/filecache"
 	"golang.org/x/tools/gopls/internal/lsp/protocol"
 	"golang.org/x/tools/gopls/internal/lsp/source"
+	"golang.org/x/tools/internal/event"
 )
 
 type stats struct {
 	app *Application
+
+	Anon bool `flag:"anon" help:"hide any fields that may contain user names, file names, or source code"`
 }
 
 func (s *stats) Name() string      { return "stats" }
@@ -41,14 +46,17 @@ Load the workspace for the current directory, and output a JSON summary of
 workspace information relevant to performance. As a side effect, this command
 populates the gopls file cache for the current workspace.
 
+By default, this command may include output that refers to the location or
+content of user code. When the -anon flag is set, fields that may refer to user
+code are hidden.
+
 Example:
-  $ gopls stats
+  $ gopls stats -anon
 `)
 	printFlagDefaults(f)
 }
 
 func (s *stats) Run(ctx context.Context, args ...string) error {
-
 	// This undocumented environment variable allows
 	// the cmd integration test to trigger a call to bug.Report.
 	if msg := os.Getenv("TEST_GOPLS_BUG"); msg != "" {
@@ -65,6 +73,10 @@ func (s *stats) Run(ctx context.Context, args ...string) error {
 		return fmt.Errorf("the stats subcommand does not work with -remote")
 	}
 
+	if !s.app.Verbose {
+		event.SetExporter(nil) // don't log errors to stderr
+	}
+
 	stats := GoplsStats{
 		GOOS:         runtime.GOOS,
 		GOARCH:       runtime.GOARCH,
@@ -139,12 +151,10 @@ func (s *stats) Run(ctx context.Context, args ...string) error {
 
 	// Gather bug reports produced by any process using
 	// this executable and persisted in the cache.
-	stats.BugReports = []string{} // non-nil for JSON
 	do("Gathering bug reports", func() error {
-		cacheDir, reports := filecache.BugReports()
-		stats.CacheDir = cacheDir
-		for _, report := range reports {
-			stats.BugReports = append(stats.BugReports, string(report))
+		stats.CacheDir, stats.BugReports = filecache.BugReports()
+		if stats.BugReports == nil {
+			stats.BugReports = []goplsbug.Bug{} // non-nil for JSON
 		}
 		return nil
 	})
@@ -186,25 +196,51 @@ func (s *stats) Run(ctx context.Context, args ...string) error {
 		return err
 	}
 
-	data, err := json.MarshalIndent(stats, "", "  ")
+	// Filter JSON output to fields that are consistent with s.Anon.
+	okFields := make(map[string]interface{})
+	{
+		v := reflect.ValueOf(stats)
+		t := v.Type()
+		for i := 0; i < t.NumField(); i++ {
+			f := t.Field(i)
+			if !token.IsExported(f.Name) {
+				continue
+			}
+			if s.Anon && f.Tag.Get("anon") != "ok" {
+				// Fields that can be served with -anon must be explicitly marked as OK.
+				continue
+			}
+			vf := v.FieldByName(f.Name)
+			okFields[f.Name] = vf.Interface()
+		}
+	}
+	data, err := json.MarshalIndent(okFields, "", "  ")
 	if err != nil {
 		return err
 	}
+
 	os.Stdout.Write(data)
 	fmt.Println()
 	return nil
 }
 
+// GoplsStats holds information extracted from a gopls session in the current
+// workspace.
+//
+// Fields that should be printed with the -anon flag should be explicitly
+// marked as `anon:"ok"`. Only fields that cannot refer to user files or code
+// should be marked as such.
 type GoplsStats struct {
-	GOOS, GOARCH, GOPLSCACHE     string
-	GoVersion                    string
-	GoplsVersion                 string
-	InitialWorkspaceLoadDuration string // in time.Duration string form
+	GOOS, GOARCH                 string `anon:"ok"`
+	GOPLSCACHE                   string
+	GoVersion                    string `anon:"ok"`
+	GoplsVersion                 string `anon:"ok"`
+	InitialWorkspaceLoadDuration string `anon:"ok"` // in time.Duration string form
 	CacheDir                     string
-	BugReports                   []string
-	MemStats                     command.MemStatsResult
-	WorkspaceStats               command.WorkspaceStatsResult
-	DirStats                     dirStats
+	BugReports                   []goplsbug.Bug
+	MemStats                     command.MemStatsResult       `anon:"ok"`
+	WorkspaceStats               command.WorkspaceStatsResult `anon:"ok"`
+	DirStats                     dirStats                     `anon:"ok"`
 }
 
 type dirStats struct {

gopls/internal/lsp/cmd/test/integration_test.go

@@ -756,6 +756,18 @@ package foo
 		}
 	}
 
+	// Check that -anon suppresses fields containing user information.
+	{
+		res2 := gopls(t, tree, "stats", "-anon")
+		res2.checkExit(true)
+		var stats2 cmd.GoplsStats
+		if err := json.Unmarshal([]byte(res2.stdout), &stats2); err != nil {
+			t.Fatalf("failed to unmarshal JSON output of stats command: %v", err)
+		}
+		if got := len(stats2.BugReports); got > 0 {
+			t.Errorf("Got %d bug reports with -anon, want 0. Reports:%+v", got, stats2.BugReports)
+		}
+	}
 }
 
 // TestFix tests the 'fix' subcommand (../suggested_fix.go).

gopls/internal/lsp/cmd/usage/stats.hlp

@@ -7,5 +7,11 @@ Load the workspace for the current directory, and output a JSON summary of
 workspace information relevant to performance. As a side effect, this command
 populates the gopls file cache for the current workspace.
 
+By default, this command may include output that refers to the location or
+content of user code. When the -anon flag is set, fields that may refer to user
+code are hidden.
+
 Example:
-  $ gopls stats
+  $ gopls stats -anon
+  -anon
+    	hide any fields that may contain user names, file names, or source code

gopls/internal/lsp/filecache/filecache.go

@@ -24,6 +24,7 @@ import (
 	"bytes"
 	"crypto/sha256"
 	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -533,17 +534,21 @@ func init() {
 		// Wait for cache init (bugs in tests happen early).
 		_, _ = getCacheDir()
 
-		value := []byte(fmt.Sprintf("%s: %+v", time.Now().Format(time.RFC3339), bug))
-		key := sha256.Sum256(value)
-		_ = Set(bugKind, key, value)
+		data, err := json.Marshal(bug)
+		if err != nil {
+			panic(fmt.Sprintf("error marshalling bug %+v: %v", bug, err))
+		}
+
+		key := sha256.Sum256(data)
+		_ = Set(bugKind, key, data)
 	})
 }
 
 // BugReports returns a new unordered array of the contents
 // of all cached bug reports produced by this executable.
 // It also returns the location of the cache directory
 // used by this process (or "" on initialization error).
-func BugReports() (string, [][]byte) {
+func BugReports() (string, []bug.Bug) {
 	// To test this logic, run:
 	// $ TEST_GOPLS_BUG=oops gopls stats   # trigger a bug
 	// $ gopls stats                       # list the bugs
@@ -552,7 +557,7 @@ func BugReports() (string, [][]byte) {
 	if err != nil {
 		return "", nil // ignore initialization errors
 	}
-	var result [][]byte
+	var result []bug.Bug
 	_ = filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error {
 		if err != nil {
 			return nil // ignore readdir/stat errors
@@ -566,7 +571,11 @@ func BugReports() (string, [][]byte) {
 			}
 			content, err := Get(bugKind, key)
 			if err == nil { // ignore read errors
-				result = append(result, content)
+				var b bug.Bug
+				if err := json.Unmarshal(content, &b); err != nil {
+					log.Printf("error marshalling bug %q: %v", string(content), err)
+				}
+				result = append(result, b)
 			}
 		}
 		return nil