Skip to content

Commit ae9efa5

Browse files
author
Dylan Le
committed
all: merge master (c353b05) into gopls-release-branch.0.9
Add replace directive for x/tools to gopls/go.mod. For golang/go#53412 Merge List: + 2022-06-16 c353b05 internal/lsp/cache: delete checkSnapshotLocked + 2022-06-16 567c98b internal/lsp/cache: don't walk URIs to invalidate metadata + 2022-06-16 dffd645 internal/lsp/cache: only clone metadata if something changed + 2022-06-16 4ba3d22 internal/lsp/cache: clone the metadata graph when clearing ShouldLoad + 2022-06-16 39d3d49 internal/lsp/cache: use metadataGraph.Clone in snapshot.clone + 2022-06-16 8a92078 internal/lsp/cache: build a new metadata graph on load + 2022-06-16 9f38ef7 internal/lsp/cache: derive workspace packages from metadata + 2022-06-16 041035c Revert "internal/lsp/cache: reduce critical sections" + 2022-06-16 d097bc9 gopls/internal/vulncheck: include nonaffecting vulnerability info + 2022-06-16 e8b9ff1 gopls/internal/govulncheck: sync x/vuln@4eb5ba4 + 2022-06-15 654a14b internal/lsp/cache: reduce critical sections + 2022-06-14 27db7f4 gopls: update golang.org/x/vuln to latest @4eb5ba4 + 2022-06-14 c993be6 go/analysis/internal/checker: log codeFact error, remove unused action.inputs + 2022-06-14 ed27611 internal/lsp/cache: cache known subdirs pattern + 2022-06-14 ebc084a internal/lsp: add inlay hints count to test summary + 2022-06-13 c15c045 internal/lsp: enable inlay hint tests + 2022-06-13 0343989 internal/lsp: fix error message for inlay hints + 2022-06-13 a41fc98 internal/lsp/cache: use [256]byte Hash instead of hex digit string + 2022-06-10 c41ddce internal/lsp: include padding in inlay hint marker tests + 2022-06-10 5e48d26 internal/lsp: add inlay hints for composite literal names + 2022-06-10 83b0675 internal/lsp: add inlay hints for constant values + 2022-06-10 ecc1479 internal/lsp: add inlay hints for variable types + 2022-06-10 65c0181 internal/lsp: support textDocument/inlayHint for parameter names Change-Id: Idc9bcd1253bbaa32a601867f25472645b104dd05
2 parents 9157b63 + c353b05 commit ae9efa5

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+1157
-413
lines changed

go/analysis/internal/checker/checker.go

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -578,7 +578,6 @@ type action struct {
578578
deps []*action
579579
objectFacts map[objectFactKey]analysis.Fact
580580
packageFacts map[packageFactKey]analysis.Fact
581-
inputs map[*analysis.Analyzer]interface{}
582581
result interface{}
583582
diagnostics []analysis.Diagnostic
584583
err error
@@ -766,7 +765,7 @@ func inheritFacts(act, dep *action) {
766765
if serialize {
767766
encodedFact, err := codeFact(fact)
768767
if err != nil {
769-
log.Panicf("internal error: encoding of %T fact failed in %v", fact, act)
768+
log.Panicf("internal error: encoding of %T fact failed in %v: %v", fact, act, err)
770769
}
771770
fact = encodedFact
772771
}
@@ -894,7 +893,7 @@ func (act *action) exportObjectFact(obj types.Object, fact analysis.Fact) {
894893
func (act *action) allObjectFacts() []analysis.ObjectFact {
895894
facts := make([]analysis.ObjectFact, 0, len(act.objectFacts))
896895
for k := range act.objectFacts {
897-
facts = append(facts, analysis.ObjectFact{k.obj, act.objectFacts[k]})
896+
facts = append(facts, analysis.ObjectFact{Object: k.obj, Fact: act.objectFacts[k]})
898897
}
899898
return facts
900899
}
@@ -940,7 +939,7 @@ func factType(fact analysis.Fact) reflect.Type {
940939
func (act *action) allPackageFacts() []analysis.PackageFact {
941940
facts := make([]analysis.PackageFact, 0, len(act.packageFacts))
942941
for k := range act.packageFacts {
943-
facts = append(facts, analysis.PackageFact{k.pkg, act.packageFacts[k]})
942+
facts = append(facts, analysis.PackageFact{Package: k.pkg, Fact: act.packageFacts[k]})
944943
}
945944
return facts
946945
}

gopls/go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ require (
99
github.com/sergi/go-diff v1.1.0
1010
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4
1111
golang.org/x/sys v0.0.0-20220209214540-3681064d5158
12-
golang.org/x/tools v0.1.11-0.20220330174940-8e193c2ba95e
13-
golang.org/x/vuln v0.0.0-20220503210553-a5481fb0c8be
12+
golang.org/x/tools v0.1.11-0.20220523181440-ccb10502d1a5
13+
golang.org/x/vuln v0.0.0-20220613164644-4eb5ba49563c
1414
honnef.co/go/tools v0.3.0
1515
mvdan.cc/gofumpt v0.3.0
1616
mvdan.cc/xurls/v2 v2.4.0

gopls/go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,8 @@ golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
7373
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
7474
golang.org/x/vuln v0.0.0-20220503210553-a5481fb0c8be h1:jokAF1mfylAi1iTQx7C44B7vyXUcSEMw8eDv0PzNu8s=
7575
golang.org/x/vuln v0.0.0-20220503210553-a5481fb0c8be/go.mod h1:twca1SxmF6/i2wHY/mj1vLIkkHdp+nil/yA32ZOP4kg=
76+
golang.org/x/vuln v0.0.0-20220613164644-4eb5ba49563c h1:r5bbIROBQtRRgoutV8Q3sFY58VzHW6jMBYl48ANSyS4=
77+
golang.org/x/vuln v0.0.0-20220613164644-4eb5ba49563c/go.mod h1:UZshlUPxXeGUM9I14UOawXQg6yosDE9cr1vKY/DzgWo=
7678
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
7779
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
7880
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=

gopls/internal/govulncheck/source.go

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ func LoadPackages(cfg *packages.Config, patterns ...string) ([]*vulncheck.Packag
5959

6060
// Source calls vulncheck.Source on the Go source in pkgs. It returns the result
6161
// with Vulns trimmed to those that are actually called.
62+
//
63+
// This function is being used by the Go IDE team.
6264
func Source(ctx context.Context, pkgs []*vulncheck.Package, c client.Client) (*vulncheck.Result, error) {
6365
r, err := vulncheck.Source(ctx, pkgs, &vulncheck.Config{Client: c})
6466
if err != nil {
@@ -77,14 +79,21 @@ func Source(ctx context.Context, pkgs []*vulncheck.Package, c client.Client) (*v
7779

7880
// CallInfo is information about calls to vulnerable functions.
7981
type CallInfo struct {
80-
CallStacks map[*vulncheck.Vuln][]vulncheck.CallStack // all call stacks
81-
VulnGroups [][]*vulncheck.Vuln // vulns grouped by ID and package
82-
ModuleVersions map[string]string // map from module paths to versions
83-
TopPackages map[string]bool // top-level packages
82+
// CallStacks contains all call stacks to vulnerable functions.
83+
CallStacks map[*vulncheck.Vuln][]vulncheck.CallStack
84+
85+
// VulnGroups contains vulnerabilities grouped by ID and package.
86+
VulnGroups [][]*vulncheck.Vuln
87+
88+
// ModuleVersions is a map of module paths to versions.
89+
ModuleVersions map[string]string
90+
91+
// TopPackages contains the top-level packages in the call info.
92+
TopPackages map[string]bool
8493
}
8594

8695
// GetCallInfo computes call stacks and related information from a vulncheck.Result.
87-
// I also makes a set of top-level packages from pkgs.
96+
// It also makes a set of top-level packages from pkgs.
8897
func GetCallInfo(r *vulncheck.Result, pkgs []*vulncheck.Package) *CallInfo {
8998
pset := map[string]bool{}
9099
for _, p := range pkgs {

gopls/internal/regtest/bench/bench_test.go

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"flag"
99
"fmt"
1010
"os"
11+
"runtime"
1112
"runtime/pprof"
1213
"testing"
1314

@@ -66,6 +67,7 @@ func TestBenchmarkIWL(t *testing.T) {
6667
results := testing.Benchmark(func(b *testing.B) {
6768
for i := 0; i < b.N; i++ {
6869
WithOptions(opts...).Run(t, "", func(t *testing.T, env *Env) {})
70+
6971
}
7072
})
7173

@@ -192,3 +194,31 @@ func TestBenchmarkDidChange(t *testing.T) {
192194
printBenchmarkResults(result)
193195
})
194196
}
197+
198+
// TestPrintMemStats measures the memory usage of loading a project.
199+
// It uses the same -didchange_dir flag as above.
200+
// Always run it in isolation since it measures global heap usage.
201+
//
202+
// Kubernetes example:
203+
// $ go test -run=TestPrintMemStats -didchange_dir=$HOME/w/kubernetes
204+
// TotalAlloc: 5766 MB
205+
// HeapAlloc: 1984 MB
206+
//
207+
// Both figures exhibit variance of less than 1%.
208+
func TestPrintMemStats(t *testing.T) {
209+
if *benchDir == "" {
210+
t.Skip("-didchange_dir is not set")
211+
}
212+
213+
// Load the program...
214+
opts := benchmarkOptions(*benchDir)
215+
WithOptions(opts...).Run(t, "", func(_ *testing.T, env *Env) {
216+
// ...and print the memory usage.
217+
runtime.GC()
218+
runtime.GC()
219+
var mem runtime.MemStats
220+
runtime.ReadMemStats(&mem)
221+
t.Logf("TotalAlloc:\t%d MB", mem.TotalAlloc/1e6)
222+
t.Logf("HeapAlloc:\t%d MB", mem.HeapAlloc/1e6)
223+
})
224+
}

gopls/internal/vulncheck/command.go

Lines changed: 80 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,15 @@ import (
1111
"context"
1212
"log"
1313
"os"
14+
"sort"
1415
"strings"
1516

1617
"golang.org/x/tools/go/packages"
1718
gvc "golang.org/x/tools/gopls/internal/govulncheck"
1819
"golang.org/x/tools/internal/lsp/command"
1920
"golang.org/x/vuln/client"
21+
"golang.org/x/vuln/osv"
22+
"golang.org/x/vuln/vulncheck"
2023
)
2124

2225
func init() {
@@ -79,29 +82,84 @@ func (c *cmd) Run(ctx context.Context, cfg *packages.Config, patterns ...string)
7982
}
8083
log.Printf("loaded %d packages\n", len(loadedPkgs))
8184

82-
r, err := gvc.Source(ctx, loadedPkgs, c.Client)
85+
log.Printf("analyzing %d packages...\n", len(loadedPkgs))
86+
87+
r, err := vulncheck.Source(ctx, loadedPkgs, &vulncheck.Config{Client: c.Client})
8388
if err != nil {
8489
return nil, err
8590
}
91+
unaffectedMods := filterUnaffected(r.Vulns)
92+
r.Vulns = filterCalled(r)
93+
8694
callInfo := gvc.GetCallInfo(r, loadedPkgs)
87-
return toVulns(callInfo)
95+
return toVulns(callInfo, unaffectedMods)
8896
// TODO: add import graphs.
8997
}
9098

91-
func toVulns(ci *gvc.CallInfo) ([]Vuln, error) {
99+
// filterCalled returns vulnerabilities where the symbols are actually called.
100+
func filterCalled(r *vulncheck.Result) []*vulncheck.Vuln {
101+
var vulns []*vulncheck.Vuln
102+
for _, v := range r.Vulns {
103+
if v.CallSink != 0 {
104+
vulns = append(vulns, v)
105+
}
106+
}
107+
return vulns
108+
}
109+
110+
// filterUnaffected returns vulnerabilities where no symbols are called,
111+
// grouped by module.
112+
func filterUnaffected(vulns []*vulncheck.Vuln) map[string][]*osv.Entry {
113+
// It is possible that the same vuln.OSV.ID has vuln.CallSink != 0
114+
// for one symbol, but vuln.CallSink == 0 for a different one, so
115+
// we need to filter out ones that have been called.
116+
called := map[string]bool{}
117+
for _, vuln := range vulns {
118+
if vuln.CallSink != 0 {
119+
called[vuln.OSV.ID] = true
120+
}
121+
}
122+
123+
modToIDs := map[string]map[string]*osv.Entry{}
124+
for _, vuln := range vulns {
125+
if !called[vuln.OSV.ID] {
126+
if _, ok := modToIDs[vuln.ModPath]; !ok {
127+
modToIDs[vuln.ModPath] = map[string]*osv.Entry{}
128+
}
129+
// keep only one vuln.OSV instance for the same ID.
130+
modToIDs[vuln.ModPath][vuln.OSV.ID] = vuln.OSV
131+
}
132+
}
133+
output := map[string][]*osv.Entry{}
134+
for m, vulnSet := range modToIDs {
135+
var vulns []*osv.Entry
136+
for _, vuln := range vulnSet {
137+
vulns = append(vulns, vuln)
138+
}
139+
sort.Slice(vulns, func(i, j int) bool { return vulns[i].ID < vulns[j].ID })
140+
output[m] = vulns
141+
}
142+
return output
143+
}
144+
145+
func fixed(v *osv.Entry) string {
146+
lf := gvc.LatestFixed(v.Affected)
147+
if lf != "" && lf[0] != 'v' {
148+
lf = "v" + lf
149+
}
150+
return lf
151+
}
152+
153+
func toVulns(ci *gvc.CallInfo, unaffectedMods map[string][]*osv.Entry) ([]Vuln, error) {
92154
var vulns []Vuln
93155

94156
for _, vg := range ci.VulnGroups {
95157
v0 := vg[0]
96-
lf := gvc.LatestFixed(v0.OSV.Affected)
97-
if lf != "" && lf[0] != 'v' {
98-
lf = "v" + lf
99-
}
100158
vuln := Vuln{
101159
ID: v0.OSV.ID,
102160
PkgPath: v0.PkgPath,
103161
CurrentVersion: ci.ModuleVersions[v0.ModPath],
104-
FixedVersion: lf,
162+
FixedVersion: fixed(v0.OSV),
105163
Details: v0.OSV.Details,
106164

107165
Aliases: v0.OSV.Aliases,
@@ -119,5 +177,19 @@ func toVulns(ci *gvc.CallInfo) ([]Vuln, error) {
119177
}
120178
vulns = append(vulns, vuln)
121179
}
180+
for m, vg := range unaffectedMods {
181+
for _, v0 := range vg {
182+
vuln := Vuln{
183+
ID: v0.ID,
184+
Details: v0.Details,
185+
Aliases: v0.Aliases,
186+
ModPath: m,
187+
URL: href(v0),
188+
CurrentVersion: "",
189+
FixedVersion: fixed(v0),
190+
}
191+
vulns = append(vulns, vuln)
192+
}
193+
}
122194
return vulns, nil
123195
}

gopls/internal/vulncheck/command_test.go

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,15 @@ func TestCmd_Run(t *testing.T) {
8181
"golang.org/bmod/bvuln.Vuln (bvuln.go:2)\n",
8282
},
8383
},
84+
{
85+
Vuln: Vuln{
86+
ID: "GO-2022-03",
87+
Details: "unaffecting vulnerability",
88+
ModPath: "golang.org/amod",
89+
URL: "https://pkg.go.dev/vuln/GO-2022-03",
90+
FixedVersion: "v1.0.4",
91+
},
92+
},
8493
}
8594
// sort reports for stability before comparison.
8695
for _, rpts := range [][]report{got, want} {
@@ -228,6 +237,21 @@ var testClient1 = &mockClient{
228237
EcosystemSpecific: osv.EcosystemSpecific{Symbols: []string{"VulnData.Vuln1", "VulnData.Vuln2"}},
229238
}},
230239
},
240+
{
241+
ID: "GO-2022-03",
242+
Details: "unaffecting vulnerability",
243+
References: []osv.Reference{
244+
{
245+
Type: "href",
246+
URL: "pkg.go.dev/vuln/GO-2022-01",
247+
},
248+
},
249+
Affected: []osv.Affected{{
250+
Package: osv.Package{Name: "golang.org/amod/avuln"},
251+
Ranges: osv.Affects{{Type: osv.TypeSemver, Events: []osv.RangeEvent{{Introduced: "1.0.0"}, {Fixed: "1.0.4"}, {Introduced: "1.1.2"}}}},
252+
EcosystemSpecific: osv.EcosystemSpecific{Symbols: []string{"nonExisting"}},
253+
}},
254+
},
231255
},
232256
"golang.org/bmod": {
233257
{

internal/lsp/cache/analysis.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ func (s *snapshot) Analyze(ctx context.Context, id string, analyzers []*source.A
5454
return results, nil
5555
}
5656

57-
type actionHandleKey string
57+
type actionHandleKey source.Hash
5858

5959
// An action represents one unit of analysis work: the application of
6060
// one analysis to one package. Actions form a DAG, both within a
@@ -170,7 +170,7 @@ func (act *actionHandle) analyze(ctx context.Context, snapshot *snapshot) ([]*so
170170
}
171171

172172
func buildActionKey(a *analysis.Analyzer, ph *packageHandle) actionHandleKey {
173-
return actionHandleKey(hashContents([]byte(fmt.Sprintf("%p %s", a, string(ph.key)))))
173+
return actionHandleKey(source.Hashf("%p%s", a, ph.key[:]))
174174
}
175175

176176
func (act *actionHandle) String() string {

internal/lsp/cache/cache.go

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ package cache
66

77
import (
88
"context"
9-
"crypto/sha256"
109
"fmt"
1110
"go/ast"
1211
"go/token"
@@ -55,7 +54,7 @@ type fileHandle struct {
5554
modTime time.Time
5655
uri span.URI
5756
bytes []byte
58-
hash string
57+
hash source.Hash
5958
err error
6059

6160
// size is the file length as reported by Stat, for the purpose of
@@ -139,7 +138,7 @@ func readFile(ctx context.Context, uri span.URI, fi os.FileInfo) (*fileHandle, e
139138
size: fi.Size(),
140139
uri: uri,
141140
bytes: data,
142-
hash: hashContents(data),
141+
hash: source.HashOf(data),
143142
}, nil
144143
}
145144

@@ -168,10 +167,6 @@ func (h *fileHandle) URI() span.URI {
168167
return h.uri
169168
}
170169

171-
func (h *fileHandle) Hash() string {
172-
return h.hash
173-
}
174-
175170
func (h *fileHandle) FileIdentity() source.FileIdentity {
176171
return source.FileIdentity{
177172
URI: h.uri,
@@ -183,16 +178,6 @@ func (h *fileHandle) Read() ([]byte, error) {
183178
return h.bytes, h.err
184179
}
185180

186-
// hashContents returns a string of hex digits denoting the hash of contents.
187-
//
188-
// TODO(adonovan): opt: use [32]byte array as a value more widely and convert
189-
// to hex digits on demand (rare). The array is larger when it appears as a
190-
// struct field (32B vs 16B) but smaller overall (string data is 64B), has
191-
// better locality, and is more efficiently hashed by runtime maps.
192-
func hashContents(contents []byte) string {
193-
return fmt.Sprintf("%64x", sha256.Sum256(contents))
194-
}
195-
196181
var cacheIndex, sessionIndex, viewIndex int64
197182

198183
func (c *Cache) ID() string { return c.id }

0 commit comments

Comments
 (0)