diff --git a/cmd/bundle/main.go b/cmd/bundle/main.go index 96cbce9a131..194797bd822 100644 --- a/cmd/bundle/main.go +++ b/cmd/bundle/main.go @@ -84,6 +84,7 @@ import ( "os" "strconv" "strings" + "unicode" "golang.org/x/tools/go/packages" ) @@ -233,7 +234,7 @@ func bundle(src, dst, dstpkg, prefix, buildTags string) ([]byte, error) { fmt.Fprintf(&out, "// Code generated by golang.org/x/tools/cmd/bundle. DO NOT EDIT.\n") if *outputFile != "" && buildTags == "" { - fmt.Fprintf(&out, "//go:generate bundle %s\n", strings.Join(os.Args[1:], " ")) + fmt.Fprintf(&out, "//go:generate bundle %s\n", strings.Join(quoteArgs(os.Args[1:]), " ")) } else { fmt.Fprintf(&out, "// $ bundle %s\n", strings.Join(os.Args[1:], " ")) } @@ -447,6 +448,35 @@ func printSameLineComment(out *bytes.Buffer, comments []*ast.CommentGroup, fset return pos } +func quoteArgs(ss []string) []string { + // From go help generate: + // + // > The arguments to the directive are space-separated tokens or + // > double-quoted strings passed to the generator as individual + // > arguments when it is run. + // + // > Quoted strings use Go syntax and are evaluated before execution; a + // > quoted string appears as a single argument to the generator. + // + var qs []string + for _, s := range ss { + if s == "" || containsSpace(s) { + s = strconv.Quote(s) + } + qs = append(qs, s) + } + return qs +} + +func containsSpace(s string) bool { + for _, r := range s { + if unicode.IsSpace(r) { + return true + } + } + return false +} + type flagFunc func(string) func (f flagFunc) Set(s string) error { diff --git a/go.mod b/go.mod index cfc184e5fb3..b46da5396a5 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,9 @@ go 1.18 // tagx:compat 1.16 require ( github.com/yuin/goldmark v1.4.13 - golang.org/x/mod v0.6.0 - golang.org/x/net v0.1.0 - golang.org/x/sys v0.1.0 + golang.org/x/mod v0.7.0 + golang.org/x/net v0.2.0 + golang.org/x/sys v0.2.0 ) + +require golang.org/x/sync v0.1.0 diff --git a/go.sum b/go.sum index da2cda7d75f..92f0a74888d 100644 --- a/go.sum +++ b/go.sum @@ -2,27 +2,28 @@ github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= diff --git a/go/analysis/passes/asmdecl/asmdecl.go b/go/analysis/passes/asmdecl/asmdecl.go index 6fbfe7e181c..7288559fc0e 100644 --- a/go/analysis/passes/asmdecl/asmdecl.go +++ b/go/analysis/passes/asmdecl/asmdecl.go @@ -92,7 +92,7 @@ var ( asmArchMips64LE = asmArch{name: "mips64le", bigEndian: false, stack: "R29", lr: true} asmArchPpc64 = asmArch{name: "ppc64", bigEndian: true, stack: "R1", lr: true, retRegs: []string{"R3", "F1"}} asmArchPpc64LE = asmArch{name: "ppc64le", bigEndian: false, stack: "R1", lr: true, retRegs: []string{"R3", "F1"}} - asmArchRISCV64 = asmArch{name: "riscv64", bigEndian: false, stack: "SP", lr: true} + asmArchRISCV64 = asmArch{name: "riscv64", bigEndian: false, stack: "SP", lr: true, retRegs: []string{"X10", "F10"}} asmArchS390X = asmArch{name: "s390x", bigEndian: true, stack: "R15", lr: true} asmArchWasm = asmArch{name: "wasm", bigEndian: false, stack: "SP", lr: false} diff --git a/go/analysis/passes/asmdecl/asmdecl_test.go b/go/analysis/passes/asmdecl/asmdecl_test.go index f6b01a9c308..50938a07571 100644 --- a/go/analysis/passes/asmdecl/asmdecl_test.go +++ b/go/analysis/passes/asmdecl/asmdecl_test.go @@ -19,11 +19,12 @@ var goosarches = []string{ "linux/arm", // asm3.s // TODO: skip test on loong64 until go toolchain supported loong64. // "linux/loong64", // asm10.s - "linux/mips64", // asm5.s - "linux/s390x", // asm6.s - "linux/ppc64", // asm7.s - "linux/mips", // asm8.s, - "js/wasm", // asm9.s + "linux/mips64", // asm5.s + "linux/s390x", // asm6.s + "linux/ppc64", // asm7.s + "linux/mips", // asm8.s, + "js/wasm", // asm9.s + "linux/riscv64", // asm11.s } func Test(t *testing.T) { diff --git a/go/analysis/passes/asmdecl/testdata/src/a/asm11.s b/go/analysis/passes/asmdecl/testdata/src/a/asm11.s new file mode 100644 index 00000000000..e81e8ee179f --- /dev/null +++ b/go/analysis/passes/asmdecl/testdata/src/a/asm11.s @@ -0,0 +1,13 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build riscv64 + +// writing to result in ABIInternal function +TEXT ·returnABIInternal(SB), NOSPLIT, $8 + MOV $123, X10 + RET +TEXT ·returnmissingABIInternal(SB), NOSPLIT, $8 + MOV $123, X20 + RET // want `RET without writing to result register` diff --git a/go/analysis/passes/loopclosure/loopclosure.go b/go/analysis/passes/loopclosure/loopclosure.go index 35fe15c9a20..bb0715c02b5 100644 --- a/go/analysis/passes/loopclosure/loopclosure.go +++ b/go/analysis/passes/loopclosure/loopclosure.go @@ -14,7 +14,6 @@ import ( "golang.org/x/tools/go/analysis/passes/inspect" "golang.org/x/tools/go/ast/inspector" "golang.org/x/tools/go/types/typeutil" - "golang.org/x/tools/internal/analysisinternal" ) const Doc = `check references to loop variables from within nested functions @@ -24,10 +23,11 @@ literal inside the loop body. It checks for patterns where access to a loop variable is known to escape the current loop iteration: 1. a call to go or defer at the end of the loop body 2. a call to golang.org/x/sync/errgroup.Group.Go at the end of the loop body + 3. a call testing.T.Run where the subtest body invokes t.Parallel() -The analyzer only considers references in the last statement of the loop body -as it is not deep enough to understand the effects of subsequent statements -which might render the reference benign. +In the case of (1) and (2), the analyzer only considers references in the last +statement of the loop body as it is not deep enough to understand the effects +of subsequent statements which might render the reference benign. For example: @@ -39,10 +39,6 @@ For example: See: https://golang.org/doc/go_faq.html#closures_and_goroutines` -// TODO(rfindley): enable support for checking parallel subtests, pending -// investigation, adding: -// 3. a call testing.T.Run where the subtest body invokes t.Parallel() - var Analyzer = &analysis.Analyzer{ Name: "loopclosure", Doc: Doc, @@ -121,7 +117,7 @@ func run(pass *analysis.Pass) (interface{}, error) { if i == lastStmt { stmts = litStmts(goInvoke(pass.TypesInfo, call)) } - if stmts == nil && analysisinternal.LoopclosureParallelSubtests { + if stmts == nil { stmts = parallelSubtest(pass.TypesInfo, call) } } @@ -178,15 +174,19 @@ func goInvoke(info *types.Info, call *ast.CallExpr) ast.Expr { return call.Args[0] } -// parallelSubtest returns statements that would would be executed -// asynchronously via the go test runner, as t.Run has been invoked with a +// parallelSubtest returns statements that can be easily proven to execute +// concurrently via the go test runner, as t.Run has been invoked with a // function literal that calls t.Parallel. // // In practice, users rely on the fact that statements before the call to // t.Parallel are synchronous. For example by declaring test := test inside the // function literal, but before the call to t.Parallel. // -// Therefore, we only flag references that occur after the call to t.Parallel: +// Therefore, we only flag references in statements that are obviously +// dominated by a call to t.Parallel. As a simple heuristic, we only consider +// statements following the final labeled statement in the function body, to +// avoid scenarios where a jump would cause either the call to t.Parallel or +// the problematic reference to be skipped. // // import "testing" // @@ -210,17 +210,81 @@ func parallelSubtest(info *types.Info, call *ast.CallExpr) []ast.Stmt { return nil } - for i, stmt := range lit.Body.List { + // Capture the *testing.T object for the first argument to the function + // literal. + if len(lit.Type.Params.List[0].Names) == 0 { + return nil + } + + tObj := info.Defs[lit.Type.Params.List[0].Names[0]] + if tObj == nil { + return nil + } + + // Match statements that occur after a call to t.Parallel following the final + // labeled statement in the function body. + // + // We iterate over lit.Body.List to have a simple, fast and "frequent enough" + // dominance relationship for t.Parallel(): lit.Body.List[i] dominates + // lit.Body.List[j] for i < j unless there is a jump. + var stmts []ast.Stmt + afterParallel := false + for _, stmt := range lit.Body.List { + stmt, labeled := unlabel(stmt) + if labeled { + // Reset: naively we don't know if a jump could have caused the + // previously considered statements to be skipped. + stmts = nil + afterParallel = false + } + + if afterParallel { + stmts = append(stmts, stmt) + continue + } + + // Check if stmt is a call to t.Parallel(), for the correct t. exprStmt, ok := stmt.(*ast.ExprStmt) if !ok { continue } - if isMethodCall(info, exprStmt.X, "testing", "T", "Parallel") { - return lit.Body.List[i+1:] + expr := exprStmt.X + if isMethodCall(info, expr, "testing", "T", "Parallel") { + call, _ := expr.(*ast.CallExpr) + if call == nil { + continue + } + x, _ := call.Fun.(*ast.SelectorExpr) + if x == nil { + continue + } + id, _ := x.X.(*ast.Ident) + if id == nil { + continue + } + if info.Uses[id] == tObj { + afterParallel = true + } } } - return nil + return stmts +} + +// unlabel returns the inner statement for the possibly labeled statement stmt, +// stripping any (possibly nested) *ast.LabeledStmt wrapper. +// +// The second result reports whether stmt was an *ast.LabeledStmt. +func unlabel(stmt ast.Stmt) (ast.Stmt, bool) { + labeled := false + for { + labelStmt, ok := stmt.(*ast.LabeledStmt) + if !ok { + return stmt, labeled + } + labeled = true + stmt = labelStmt.Stmt + } } // isMethodCall reports whether expr is a method call of diff --git a/go/analysis/passes/loopclosure/loopclosure_test.go b/go/analysis/passes/loopclosure/loopclosure_test.go index 8bf1d7a7bcc..55fb2a4a3d6 100644 --- a/go/analysis/passes/loopclosure/loopclosure_test.go +++ b/go/analysis/passes/loopclosure/loopclosure_test.go @@ -9,23 +9,14 @@ import ( "golang.org/x/tools/go/analysis/analysistest" "golang.org/x/tools/go/analysis/passes/loopclosure" - "golang.org/x/tools/internal/analysisinternal" "golang.org/x/tools/internal/typeparams" ) func Test(t *testing.T) { testdata := analysistest.TestData() - tests := []string{"a", "golang.org/..."} + tests := []string{"a", "golang.org/...", "subtests"} if typeparams.Enabled { tests = append(tests, "typeparams") } analysistest.Run(t, testdata, loopclosure.Analyzer, tests...) - - // Enable checking of parallel subtests. - defer func(parallelSubtest bool) { - analysisinternal.LoopclosureParallelSubtests = parallelSubtest - }(analysisinternal.LoopclosureParallelSubtests) - analysisinternal.LoopclosureParallelSubtests = true - - analysistest.Run(t, testdata, loopclosure.Analyzer, "subtests") } diff --git a/go/analysis/passes/loopclosure/testdata/src/subtests/subtest.go b/go/analysis/passes/loopclosure/testdata/src/subtests/subtest.go index 2a97244a1a5..c95fa1f0b1e 100644 --- a/go/analysis/passes/loopclosure/testdata/src/subtests/subtest.go +++ b/go/analysis/passes/loopclosure/testdata/src/subtests/subtest.go @@ -47,6 +47,14 @@ func _(t *testing.T) { println(test) // want "loop variable test captured by func literal" }) + // Check that *testing.T value matters. + t.Run("", func(t *testing.T) { + var x testing.T + x.Parallel() + println(i) + println(test) + }) + // Check that shadowing the loop variables within the test literal is OK if // it occurs before t.Parallel(). t.Run("", func(t *testing.T) { @@ -92,6 +100,46 @@ func _(t *testing.T) { println(i) println(test) }) + + // Check that there is no diagnostic when a jump to a label may have caused + // the call to t.Parallel to have been skipped. + t.Run("", func(t *testing.T) { + if true { + goto Test + } + t.Parallel() + Test: + println(i) + println(test) + }) + + // Check that there is no diagnostic when a jump to a label may have caused + // the loop variable reference to be skipped, but there is a diagnostic + // when both the call to t.Parallel and the loop variable reference occur + // after the final label in the block. + t.Run("", func(t *testing.T) { + if true { + goto Test + } + t.Parallel() + println(i) // maybe OK + Test: + t.Parallel() + println(test) // want "loop variable test captured by func literal" + }) + + // Check that multiple labels are handled. + t.Run("", func(t *testing.T) { + if true { + goto Test1 + } else { + goto Test2 + } + Test1: + Test2: + t.Parallel() + println(test) // want "loop variable test captured by func literal" + }) } } @@ -119,3 +167,32 @@ func _(t *T) { }) } } + +// Check that the top-level must be parallel in order to cause a diagnostic. +// +// From https://pkg.go.dev/testing: +// +// "Run does not return until parallel subtests have completed, providing a +// way to clean up after a group of parallel tests" +func _(t *testing.T) { + for _, test := range []int{1, 2, 3} { + // In this subtest, a/b must complete before the synchronous subtest "a" + // completes, so the reference to test does not escape the current loop + // iteration. + t.Run("a", func(s *testing.T) { + s.Run("b", func(u *testing.T) { + u.Parallel() + println(test) + }) + }) + + // In this subtest, c executes concurrently, so the reference to test may + // escape the current loop iteration. + t.Run("c", func(s *testing.T) { + s.Parallel() + s.Run("d", func(u *testing.T) { + println(test) // want "loop variable test captured by func literal" + }) + }) + } +} diff --git a/go/analysis/passes/tests/testdata/src/a/go118_test.go b/go/analysis/passes/tests/testdata/src/a/go118_test.go index dc898daca0b..e2bc3f3a0bd 100644 --- a/go/analysis/passes/tests/testdata/src/a/go118_test.go +++ b/go/analysis/passes/tests/testdata/src/a/go118_test.go @@ -94,3 +94,8 @@ func FuzzObjectMethod(f *testing.F) { } f.Fuzz(obj.myVar) // ok } + +// Test for golang/go#56505: checking fuzz arguments should not panic on *error. +func FuzzIssue56505(f *testing.F) { + f.Fuzz(func(e *error) {}) // want "the first parameter of a fuzz target must be \\*testing.T" +} diff --git a/go/analysis/passes/tests/tests.go b/go/analysis/passes/tests/tests.go index cab2fa20fa5..935aad00c98 100644 --- a/go/analysis/passes/tests/tests.go +++ b/go/analysis/passes/tests/tests.go @@ -269,7 +269,9 @@ func isTestingType(typ types.Type, testingType string) bool { if !ok { return false } - return named.Obj().Pkg().Path() == "testing" && named.Obj().Name() == testingType + obj := named.Obj() + // obj.Pkg is nil for the error type. + return obj != nil && obj.Pkg() != nil && obj.Pkg().Path() == "testing" && obj.Name() == testingType } // Validate that fuzz target function's arguments are of accepted types. diff --git a/go/analysis/unitchecker/unitchecker.go b/go/analysis/unitchecker/unitchecker.go index 9827b57f529..d9c8f11cdd4 100644 --- a/go/analysis/unitchecker/unitchecker.go +++ b/go/analysis/unitchecker/unitchecker.go @@ -50,7 +50,7 @@ import ( "golang.org/x/tools/go/analysis" "golang.org/x/tools/go/analysis/internal/analysisflags" - "golang.org/x/tools/go/analysis/internal/facts" + "golang.org/x/tools/internal/facts" "golang.org/x/tools/internal/typeparams" ) @@ -287,13 +287,13 @@ func run(fset *token.FileSet, cfg *Config, analyzers []*analysis.Analyzer) ([]re analyzers = filtered // Read facts from imported packages. - read := func(path string) ([]byte, error) { - if vetx, ok := cfg.PackageVetx[path]; ok { + read := func(imp *types.Package) ([]byte, error) { + if vetx, ok := cfg.PackageVetx[imp.Path()]; ok { return ioutil.ReadFile(vetx) } return nil, nil // no .vetx file, no facts } - facts, err := facts.Decode(pkg, read) + facts, err := facts.NewDecoder(pkg).Decode(read) if err != nil { return nil, err } diff --git a/go/callgraph/vta/testdata/src/function_alias.go b/go/callgraph/vta/testdata/src/function_alias.go index b38e0e00d69..0a8dffe79d4 100644 --- a/go/callgraph/vta/testdata/src/function_alias.go +++ b/go/callgraph/vta/testdata/src/function_alias.go @@ -33,42 +33,42 @@ func Baz(f func()) { // t2 = *t1 // *t2 = Baz$1 // t3 = local A (a) -// t4 = &t3.foo [#0] -// t5 = *t1 -// t6 = *t5 -// *t4 = t6 +// t4 = *t1 +// t5 = *t4 +// t6 = &t3.foo [#0] +// *t6 = t5 // t7 = &t3.foo [#0] // t8 = *t7 // t9 = t8() -// t10 = &t3.do [#1] *Doer -// t11 = &t3.foo [#0] *func() -// t12 = *t11 func() -// t13 = changetype Doer <- func() (t12) Doer -// *t10 = t13 +// t10 = &t3.foo [#0] *func() +// t11 = *t10 func() +// t12 = &t3.do [#1] *Doer +// t13 = changetype Doer <- func() (t11) Doer +// *t12 = t13 // t14 = &t3.do [#1] *Doer // t15 = *t14 Doer // t16 = t15() () // Flow chain showing that Baz$1 reaches t8(): -// Baz$1 -> t2 <-> PtrFunction(func()) <-> t5 -> t6 -> t4 <-> Field(testdata.A:foo) <-> t7 -> t8 +// Baz$1 -> t2 <-> PtrFunction(func()) <-> t4 -> t5 -> t6 <-> Field(testdata.A:foo) <-> t7 -> t8 // Flow chain showing that Baz$1 reaches t15(): -// Field(testdata.A:foo) <-> t11 -> t12 -> t13 -> t10 <-> Field(testdata.A:do) <-> t14 -> t15 +// Field(testdata.A:foo) <-> t10 -> t11 -> t13 -> t12 <-> Field(testdata.A:do) <-> t14 -> t15 // WANT: // Local(f) -> Local(t0) // Local(t0) -> PtrFunction(func()) // Function(Baz$1) -> Local(t2) -// PtrFunction(func()) -> Local(t0), Local(t2), Local(t5) +// PtrFunction(func()) -> Local(t0), Local(t2), Local(t4) // Local(t2) -> PtrFunction(func()) -// Local(t4) -> Field(testdata.A:foo) -// Local(t5) -> Local(t6), PtrFunction(func()) -// Local(t6) -> Local(t4) +// Local(t6) -> Field(testdata.A:foo) +// Local(t4) -> Local(t5), PtrFunction(func()) +// Local(t5) -> Local(t6) // Local(t7) -> Field(testdata.A:foo), Local(t8) -// Field(testdata.A:foo) -> Local(t11), Local(t4), Local(t7) -// Local(t4) -> Field(testdata.A:foo) -// Field(testdata.A:do) -> Local(t10), Local(t14) -// Local(t10) -> Field(testdata.A:do) -// Local(t11) -> Field(testdata.A:foo), Local(t12) -// Local(t12) -> Local(t13) -// Local(t13) -> Local(t10) +// Field(testdata.A:foo) -> Local(t10), Local(t6), Local(t7) +// Local(t6) -> Field(testdata.A:foo) +// Field(testdata.A:do) -> Local(t12), Local(t14) +// Local(t12) -> Field(testdata.A:do) +// Local(t10) -> Field(testdata.A:foo), Local(t11) +// Local(t11) -> Local(t13) +// Local(t13) -> Local(t12) // Local(t14) -> Field(testdata.A:do), Local(t15) diff --git a/go/gcexportdata/gcexportdata.go b/go/gcexportdata/gcexportdata.go index 42adb8f697b..620446207e2 100644 --- a/go/gcexportdata/gcexportdata.go +++ b/go/gcexportdata/gcexportdata.go @@ -30,7 +30,7 @@ import ( "io/ioutil" "os/exec" - "golang.org/x/tools/go/internal/gcimporter" + "golang.org/x/tools/internal/gcimporter" ) // Find returns the name of an object (.o) or archive (.a) file diff --git a/go/ssa/builder.go b/go/ssa/builder.go index 98ed49dfead..8ec8f6e310b 100644 --- a/go/ssa/builder.go +++ b/go/ssa/builder.go @@ -453,12 +453,16 @@ func (b *builder) addr(fn *Function, e ast.Expr, escaping bool) lvalue { } wantAddr := true v := b.receiver(fn, e.X, wantAddr, escaping, sel) - last := len(sel.index) - 1 - return &address{ - addr: emitFieldSelection(fn, v, sel.index[last], true, e.Sel), - pos: e.Sel.Pos(), - expr: e.Sel, + index := sel.index[len(sel.index)-1] + fld := typeparams.CoreType(deref(v.Type())).(*types.Struct).Field(index) + + // Due to the two phases of resolving AssignStmt, a panic from x.f = p() + // when x is nil is required to come after the side-effects of + // evaluating x and p(). + emit := func(fn *Function) Value { + return emitFieldSelection(fn, v, index, true, e.Sel) } + return &lazyAddress{addr: emit, t: fld.Type(), pos: e.Sel.Pos(), expr: e.Sel} case *ast.IndexExpr: var x Value @@ -487,13 +491,19 @@ func (b *builder) addr(fn *Function, e ast.Expr, escaping bool) lvalue { if isUntyped(index.Type()) { index = emitConv(fn, index, tInt) } - v := &IndexAddr{ - X: x, - Index: index, + // Due to the two phases of resolving AssignStmt, a panic from x[i] = p() + // when x is nil or i is out-of-bounds is required to come after the + // side-effects of evaluating x, i and p(). + emit := func(fn *Function) Value { + v := &IndexAddr{ + X: x, + Index: index, + } + v.setPos(e.Lbrack) + v.setType(et) + return fn.emit(v) } - v.setPos(e.Lbrack) - v.setType(et) - return &address{addr: fn.emit(v), pos: e.Lbrack, expr: e} + return &lazyAddress{addr: emit, t: deref(et), pos: e.Lbrack, expr: e} case *ast.StarExpr: return &address{addr: b.expr(fn, e.X), pos: e.Star, expr: e} @@ -669,6 +679,8 @@ func (b *builder) expr0(fn *Function, e ast.Expr, tv types.TypeAndValue) Value { y.pos = e.Lparen case *SliceToArrayPointer: y.pos = e.Lparen + case *UnOp: // conversion from slice to array. + y.pos = e.Lparen } } return y diff --git a/go/ssa/builder_go117_test.go b/go/ssa/builder_go117_test.go index 46a09526f59..69985970596 100644 --- a/go/ssa/builder_go117_test.go +++ b/go/ssa/builder_go117_test.go @@ -57,8 +57,6 @@ func TestBuildPackageFailuresGo117(t *testing.T) { importer types.Importer }{ {"slice to array pointer - source is not a slice", "package p; var s [4]byte; var _ = (*[4]byte)(s)", nil}, - // TODO(taking) re-enable test below for Go versions < Go 1.20 - see issue #54822 - // {"slice to array pointer - dest is not a pointer", "package p; var s []byte; var _ = ([4]byte)(s)", nil}, {"slice to array pointer - dest pointer elem is not an array", "package p; var s []byte; var _ = (*byte)(s)", nil}, } diff --git a/go/ssa/builder_go120_test.go b/go/ssa/builder_go120_test.go new file mode 100644 index 00000000000..84bdd4c41ab --- /dev/null +++ b/go/ssa/builder_go120_test.go @@ -0,0 +1,48 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.20 +// +build go1.20 + +package ssa_test + +import ( + "go/ast" + "go/parser" + "go/token" + "go/types" + "testing" + + "golang.org/x/tools/go/ssa" + "golang.org/x/tools/go/ssa/ssautil" +) + +func TestBuildPackageGo120(t *testing.T) { + tests := []struct { + name string + src string + importer types.Importer + }{ + {"slice to array", "package p; var s []byte; var _ = ([4]byte)(s)", nil}, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "p.go", tc.src, parser.ParseComments) + if err != nil { + t.Error(err) + } + files := []*ast.File{f} + + pkg := types.NewPackage("p", "") + conf := &types.Config{Importer: tc.importer} + if _, _, err := ssautil.BuildPackage(conf, fset, pkg, files, ssa.SanityCheckFunctions); err != nil { + t.Errorf("unexpected error: %v", err) + } + }) + } +} diff --git a/go/ssa/emit.go b/go/ssa/emit.go index fb11c3558d3..f6537acc97f 100644 --- a/go/ssa/emit.go +++ b/go/ssa/emit.go @@ -177,7 +177,6 @@ func emitConv(f *Function, val Value, typ types.Type) Value { if types.Identical(t_src, typ) { return val } - ut_dst := typ.Underlying() ut_src := t_src.Underlying() @@ -229,12 +228,32 @@ func emitConv(f *Function, val Value, typ types.Type) Value { // Conversion from slice to array pointer? if slice, ok := ut_src.(*types.Slice); ok { - if ptr, ok := ut_dst.(*types.Pointer); ok { + switch t := ut_dst.(type) { + case *types.Pointer: + ptr := t if arr, ok := ptr.Elem().Underlying().(*types.Array); ok && types.Identical(slice.Elem(), arr.Elem()) { c := &SliceToArrayPointer{X: val} - c.setType(ut_dst) + // TODO(taking): Check if this should be ut_dst or ptr. + c.setType(ptr) return f.emit(c) } + case *types.Array: + arr := t + if arr.Len() == 0 { + return zeroValue(f, arr) + } + if types.Identical(slice.Elem(), arr.Elem()) { + c := &SliceToArrayPointer{X: val} + c.setType(types.NewPointer(arr)) + x := f.emit(c) + unOp := &UnOp{ + Op: token.MUL, + X: x, + CommaOk: false, + } + unOp.setType(typ) + return f.emit(unOp) + } } } // A representation-changing conversion? diff --git a/go/ssa/interp/interp_go120_test.go b/go/ssa/interp/interp_go120_test.go new file mode 100644 index 00000000000..d8eb2c21341 --- /dev/null +++ b/go/ssa/interp/interp_go120_test.go @@ -0,0 +1,12 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.20 +// +build go1.20 + +package interp_test + +func init() { + testdataTests = append(testdataTests, "slice2array.go") +} diff --git a/go/ssa/interp/interp_test.go b/go/ssa/interp/interp_test.go index a0acf2f968a..51a74015c95 100644 --- a/go/ssa/interp/interp_test.go +++ b/go/ssa/interp/interp_test.go @@ -127,6 +127,7 @@ var testdataTests = []string{ "width32.go", "fixedbugs/issue52342.go", + "fixedbugs/issue55086.go", } func init() { diff --git a/go/ssa/interp/testdata/fixedbugs/issue55086.go b/go/ssa/interp/testdata/fixedbugs/issue55086.go new file mode 100644 index 00000000000..84c81e91a26 --- /dev/null +++ b/go/ssa/interp/testdata/fixedbugs/issue55086.go @@ -0,0 +1,132 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func a() (r string) { + s := "initial" + var p *struct{ i int } + defer func() { + recover() + r = s + }() + + s, p.i = "set", 2 // s must be set before p.i panics + return "unreachable" +} + +func b() (r string) { + s := "initial" + fn := func() []int { panic("") } + defer func() { + recover() + r = s + }() + + s, fn()[0] = "set", 2 // fn() panics before any assignment occurs + return "unreachable" +} + +func c() (r string) { + s := "initial" + var p map[int]int + defer func() { + recover() + r = s + }() + + s, p[0] = "set", 2 //s must be set before p[0] index panics" + return "unreachable" +} + +func d() (r string) { + s := "initial" + var p map[int]int + defer func() { + recover() + r = s + }() + fn := func() int { panic("") } + + s, p[0] = "set", fn() // fn() panics before s is set + return "unreachable" +} + +func e() (r string) { + s := "initial" + p := map[int]int{} + defer func() { + recover() + r = s + }() + fn := func() int { panic("") } + + s, p[fn()] = "set", 0 // fn() panics before any assignment occurs + return "unreachable" +} + +func f() (r string) { + s := "initial" + p := []int{} + defer func() { + recover() + r = s + }() + + s, p[1] = "set", 0 // p[1] panics after s is set + return "unreachable" +} + +func g() (r string) { + s := "initial" + p := map[any]any{} + defer func() { + recover() + r = s + }() + var i any = func() {} + s, p[i] = "set", 0 // p[i] panics after s is set + return "unreachable" +} + +func h() (r string) { + fail := false + defer func() { + recover() + if fail { + r = "fail" + } else { + r = "success" + } + }() + + type T struct{ f int } + var p *struct{ *T } + + // The implicit "p.T" operand should be evaluated in phase 1 (and panic), + // before the "fail = true" assignment in phase 2. + fail, p.f = true, 0 + return "unreachable" +} + +func main() { + for _, test := range []struct { + fn func() string + want string + desc string + }{ + {a, "set", "s must be set before p.i panics"}, + {b, "initial", "p() panics before s is set"}, + {c, "set", "s must be set before p[0] index panics"}, + {d, "initial", "fn() panics before s is set"}, + {e, "initial", "fn() panics before s is set"}, + {f, "set", "p[1] panics after s is set"}, + {g, "set", "p[i] panics after s is set"}, + {h, "success", "p.T panics before fail is set"}, + } { + if test.fn() != test.want { + panic(test.desc) + } + } +} diff --git a/go/ssa/interp/testdata/slice2array.go b/go/ssa/interp/testdata/slice2array.go new file mode 100644 index 00000000000..43c0543eabf --- /dev/null +++ b/go/ssa/interp/testdata/slice2array.go @@ -0,0 +1,56 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test for slice to array conversion introduced in go1.20 +// See: https://tip.golang.org/ref/spec#Conversions_from_slice_to_array_pointer + +package main + +func main() { + s := make([]byte, 3, 4) + s[0], s[1], s[2] = 2, 3, 5 + a := ([2]byte)(s) + s[0] = 7 + + if a != [2]byte{2, 3} { + panic("converted from non-nil slice to array") + } + + { + var s []int + a:= ([0]int)(s) + if a != [0]int{} { + panic("zero len array is not equal") + } + } + + if emptyToEmptyDoesNotPanic() { + panic("no panic expected from emptyToEmptyDoesNotPanic()") + } + if !threeToFourDoesPanic() { + panic("panic expected from threeToFourDoesPanic()") + } +} + +func emptyToEmptyDoesNotPanic() (raised bool) { + defer func() { + if e := recover(); e != nil { + raised = true + } + }() + var s []int + _ = ([0]int)(s) + return false +} + +func threeToFourDoesPanic() (raised bool) { + defer func() { + if e := recover(); e != nil { + raised = true + } + }() + s := make([]int, 3, 5) + _ = ([4]int)(s) + return false +} \ No newline at end of file diff --git a/go/ssa/lvalue.go b/go/ssa/lvalue.go index 64262def8b2..455b1e50fa4 100644 --- a/go/ssa/lvalue.go +++ b/go/ssa/lvalue.go @@ -93,6 +93,42 @@ func (e *element) typ() types.Type { return e.t } +// A lazyAddress is an lvalue whose address is the result of an instruction. +// These work like an *address except a new address.address() Value +// is created on each load, store and address call. +// A lazyAddress can be used to control when a side effect (nil pointer +// dereference, index out of bounds) of using a location happens. +type lazyAddress struct { + addr func(fn *Function) Value // emit to fn the computation of the address + t types.Type // type of the location + pos token.Pos // source position + expr ast.Expr // source syntax of the value (not address) [debug mode] +} + +func (l *lazyAddress) load(fn *Function) Value { + load := emitLoad(fn, l.addr(fn)) + load.pos = l.pos + return load +} + +func (l *lazyAddress) store(fn *Function, v Value) { + store := emitStore(fn, l.addr(fn), v, l.pos) + if l.expr != nil { + // store.Val is v, converted for assignability. + emitDebugRef(fn, l.expr, store.Val, false) + } +} + +func (l *lazyAddress) address(fn *Function) Value { + addr := l.addr(fn) + if l.expr != nil { + emitDebugRef(fn, l.expr, addr, true) + } + return addr +} + +func (l *lazyAddress) typ() types.Type { return l.t } + // A blank is a dummy variable whose name is "_". // It is not reified: loads are illegal and stores are ignored. type blank struct{} diff --git a/go/types/typeutil/map.go b/go/types/typeutil/map.go index dcc029b8733..7bd2fdb38be 100644 --- a/go/types/typeutil/map.go +++ b/go/types/typeutil/map.go @@ -332,7 +332,9 @@ func (h Hasher) hashFor(t types.Type) uint32 { // Method order is not significant. // Ignore m.Pkg(). m := t.Method(i) - hash += 3*hashString(m.Name()) + 5*h.Hash(m.Type()) + // Use shallow hash on method signature to + // avoid anonymous interface cycles. + hash += 3*hashString(m.Name()) + 5*h.shallowHash(m.Type()) } // Hash type restrictions. @@ -434,3 +436,76 @@ func (h Hasher) hashPtr(ptr interface{}) uint32 { h.ptrMap[ptr] = hash return hash } + +// shallowHash computes a hash of t without looking at any of its +// element Types, to avoid potential anonymous cycles in the types of +// interface methods. +// +// When an unnamed non-empty interface type appears anywhere among the +// arguments or results of an interface method, there is a potential +// for endless recursion. Consider: +// +// type X interface { m() []*interface { X } } +// +// The problem is that the Methods of the interface in m's result type +// include m itself; there is no mention of the named type X that +// might help us break the cycle. +// (See comment in go/types.identical, case *Interface, for more.) +func (h Hasher) shallowHash(t types.Type) uint32 { + // t is the type of an interface method (Signature), + // its params or results (Tuples), or their immediate + // elements (mostly Slice, Pointer, Basic, Named), + // so there's no need to optimize anything else. + switch t := t.(type) { + case *types.Signature: + var hash uint32 = 604171 + if t.Variadic() { + hash *= 971767 + } + // The Signature/Tuple recursion is always finite + // and invariably shallow. + return hash + 1062599*h.shallowHash(t.Params()) + 1282529*h.shallowHash(t.Results()) + + case *types.Tuple: + n := t.Len() + hash := 9137 + 2*uint32(n) + for i := 0; i < n; i++ { + hash += 53471161 * h.shallowHash(t.At(i).Type()) + } + return hash + + case *types.Basic: + return 45212177 * uint32(t.Kind()) + + case *types.Array: + return 1524181 + 2*uint32(t.Len()) + + case *types.Slice: + return 2690201 + + case *types.Struct: + return 3326489 + + case *types.Pointer: + return 4393139 + + case *typeparams.Union: + return 562448657 + + case *types.Interface: + return 2124679 // no recursion here + + case *types.Map: + return 9109 + + case *types.Chan: + return 9127 + + case *types.Named: + return h.hashPtr(t.Obj()) + + case *typeparams.TypeParam: + return h.hashPtr(t.Obj()) + } + panic(fmt.Sprintf("shallowHash: %T: %v", t, t)) +} diff --git a/go/types/typeutil/map_test.go b/go/types/typeutil/map_test.go index 8cd643e5b48..ee73ff9cfd5 100644 --- a/go/types/typeutil/map_test.go +++ b/go/types/typeutil/map_test.go @@ -244,6 +244,14 @@ func Bar[P Constraint[P]]() {} func Baz[Q any]() {} // The underlying type of Constraint[P] is any. // But Quux is not. func Quux[Q interface{ quux() }]() {} + + +type Issue56048_I interface{ m() interface { Issue56048_I } } +var Issue56048 = Issue56048_I.m + +type Issue56048_Ib interface{ m() chan []*interface { Issue56048_Ib } } +var Issue56048b = Issue56048_Ib.m + ` fset := token.NewFileSet() @@ -296,12 +304,14 @@ func Quux[Q interface{ quux() }]() {} ME1Type = scope.Lookup("ME1Type").Type() ME2 = scope.Lookup("ME2").Type() - Constraint = scope.Lookup("Constraint").Type() - Foo = scope.Lookup("Foo").Type() - Fn = scope.Lookup("Fn").Type() - Bar = scope.Lookup("Foo").Type() - Baz = scope.Lookup("Foo").Type() - Quux = scope.Lookup("Quux").Type() + Constraint = scope.Lookup("Constraint").Type() + Foo = scope.Lookup("Foo").Type() + Fn = scope.Lookup("Fn").Type() + Bar = scope.Lookup("Foo").Type() + Baz = scope.Lookup("Foo").Type() + Quux = scope.Lookup("Quux").Type() + Issue56048 = scope.Lookup("Issue56048").Type() + Issue56048b = scope.Lookup("Issue56048b").Type() ) tmap := new(typeutil.Map) @@ -371,6 +381,9 @@ func Quux[Q interface{ quux() }]() {} {Bar, "Bar", false}, {Baz, "Baz", false}, {Quux, "Quux", true}, + + {Issue56048, "Issue56048", true}, // (not actually about generics) + {Issue56048b, "Issue56048b", true}, // (not actually about generics) } for _, step := range steps { diff --git a/gopls/README.md b/gopls/README.md index 646419b7d06..56d15921a70 100644 --- a/gopls/README.md +++ b/gopls/README.md @@ -5,56 +5,57 @@ `gopls` (pronounced "Go please") is the official Go [language server] developed by the Go team. It provides IDE features to any [LSP]-compatible editor. - + You should not need to interact with `gopls` directly--it will be automatically integrated into your editor. The specific features and settings vary slightly -by editor, so we recommend that you proceed to the [documentation for your -editor](#editors) below. +by editor, so we recommend that you proceed to the +[documentation for your editor](#editors) below. ## Editors To get started with `gopls`, install an LSP plugin in your editor of choice. -* [VSCode](https://github.com/golang/vscode-go/blob/master/README.md) +* [VS Code](https://github.com/golang/vscode-go/blob/master/README.md) * [Vim / Neovim](doc/vim.md) * [Emacs](doc/emacs.md) * [Atom](https://github.com/MordFustang21/ide-gopls) * [Sublime Text](doc/subl.md) * [Acme](https://github.com/fhs/acme-lsp) +* [Lapce](https://github.com/lapce-community/lapce-go) -If you use `gopls` with an editor that is not on this list, please let us know -by [filing an issue](#new-issue) or [modifying this documentation](doc/contributing.md). +If you use `gopls` with an editor that is not on this list, please send us a CL +[updating this documentation](doc/contributing.md). ## Installation For the most part, you should not need to install or update `gopls`. Your editor should handle that step for you. -If you do want to get the latest stable version of `gopls`, change to any -directory that is both outside of your `GOPATH` and outside of a module (a temp -directory is fine), and run: +If you do want to get the latest stable version of `gopls`, run the following +command: ```sh go install golang.org/x/tools/gopls@latest ``` -Learn more in the [advanced installation -instructions](doc/advanced.md#installing-unreleased-versions). +Learn more in the +[advanced installation instructions](doc/advanced.md#installing-unreleased-versions). + +Learn more about gopls releases in the [release policy](doc/releases.md). ## Setting up your workspace -`gopls` supports both Go module and GOPATH modes, but if you are working with -multiple modules or uncommon project layouts, you will need to specifically -configure your workspace. See the [Workspace document](doc/workspace.md) for -information on supported workspace layouts. +`gopls` supports both Go module, multi-module and GOPATH modes. See the +[workspace documentation](doc/workspace.md) for information on supported +workspace layouts. ## Configuration You can configure `gopls` to change your editor experience or view additional debugging information. Configuration options will be made available by your editor, so see your [editor's instructions](#editors) for specific details. A -full list of `gopls` settings can be found in the [Settings documentation](doc/settings.md). +full list of `gopls` settings can be found in the [settings documentation](doc/settings.md). ### Environment variables @@ -62,27 +63,36 @@ full list of `gopls` settings can be found in the [Settings documentation](doc/s variables you configure. Some editors, such as VS Code, allow users to selectively override the values of some environment variables. -## Troubleshooting +## Support Policy -If you are having issues with `gopls`, please follow the steps described in the -[troubleshooting guide](doc/troubleshooting.md). +Gopls is maintained by engineers on the +[Go tools team](https://github.com/orgs/golang/teams/tools-team/members), +who actively monitor the +[Go](https://github.com/golang/go/issues?q=is%3Aissue+is%3Aopen+label%3Agopls) +and +[VS Code Go](https://github.com/golang/vscode-go/issues) issue trackers. -## Supported Go versions and build systems +### Supported Go versions `gopls` follows the [Go Release Policy](https://golang.org/doc/devel/release.html#policy), meaning that it officially supports the last 2 major Go releases. Per -[issue #39146](golang.org/issues/39146), we attempt to maintain best-effort +[issue #39146](https://go.dev/issues/39146), we attempt to maintain best-effort support for the last 4 major Go releases, but this support extends only to not breaking the build and avoiding easily fixable regressions. -The following table shows the final gopls version that supports being built -with a given Go version. Go releases more recent than any in the table can -build any version of gopls. +In the context of this discussion, gopls "supports" a Go version if it supports +being built with that Go version as well as integrating with the `go` command +of that Go version. -| Go Version | Final gopls Version With Support | -| ----------- | -------------------------------- | +The following table shows the final gopls version that supports a given Go +version. Go releases more recent than any in the table can be used with any +version of gopls. + +| Go Version | Final gopls version with support (without warnings) | +| ----------- | --------------------------------------------------- | | Go 1.12 | [gopls@v0.7.5](https://github.com/golang/tools/releases/tag/gopls%2Fv0.7.5) | +| Go 1.15 | [gopls@v0.9.5](https://github.com/golang/tools/releases/tag/gopls%2Fv0.9.5) | Our extended support is enforced via [continuous integration with older Go versions](doc/contributing.md#ci). This legacy Go CI may not block releases: @@ -90,13 +100,22 @@ test failures may be skipped rather than fixed. Furthermore, if a regression in an older Go version causes irreconcilable CI failures, we may drop support for that Go version in CI if it is 3 or 4 Go versions old. -`gopls` currently only supports the `go` command, so if you are using a -different build system, `gopls` will not work well. Bazel is not officially -supported, but Bazel support is in development (see -[bazelbuild/rules_go#512](https://github.com/bazelbuild/rules_go/issues/512)). +### Supported build systems + +`gopls` currently only supports the `go` command, so if you are using +a different build system, `gopls` will not work well. Bazel is not officially +supported, but may be made to work with an appropriately configured +`go/packages` driver. See +[bazelbuild/rules_go#512](https://github.com/bazelbuild/rules_go/issues/512) +for more information. You can follow [these instructions](https://github.com/bazelbuild/rules_go/wiki/Editor-setup) to configure your `gopls` to work with Bazel. +### Troubleshooting + +If you are having issues with `gopls`, please follow the steps described in the +[troubleshooting guide](doc/troubleshooting.md). + ## Additional information * [Features](doc/features.md) @@ -110,4 +129,3 @@ to configure your `gopls` to work with Bazel. [language server]: https://langserver.org [LSP]: https://microsoft.github.io/language-server-protocol/ -[Gophers Slack]: https://gophers.slack.com/ diff --git a/gopls/api-diff/api_diff.go b/gopls/api-diff/api_diff.go index f39feaa6107..8bb54186bab 100644 --- a/gopls/api-diff/api_diff.go +++ b/gopls/api-diff/api_diff.go @@ -13,253 +13,77 @@ import ( "encoding/json" "flag" "fmt" - "io" - "io/ioutil" "log" "os" "os/exec" - "path/filepath" - "strings" + "github.com/google/go-cmp/cmp" "golang.org/x/tools/gopls/internal/lsp/source" - diffpkg "golang.org/x/tools/internal/diff" - "golang.org/x/tools/internal/gocommand" ) -var ( - previousVersionFlag = flag.String("prev", "", "version to compare against") - versionFlag = flag.String("version", "", "version being tagged, or current version if omitted") -) +const usage = `api-diff [] + +Compare the API of two gopls versions. If the second argument is provided, it +will be used as the new version to compare against. Otherwise, compare against +the current API. +` func main() { flag.Parse() - apiDiff, err := diffAPI(*versionFlag, *previousVersionFlag) + if flag.NArg() < 1 || flag.NArg() > 2 { + fmt.Fprint(os.Stderr, usage) + os.Exit(2) + } + + oldVer := flag.Arg(0) + newVer := "" + if flag.NArg() == 2 { + newVer = flag.Arg(1) + } + + apiDiff, err := diffAPI(oldVer, newVer) if err != nil { log.Fatal(err) } - fmt.Printf(` -%s -`, apiDiff) -} - -type JSON interface { - String() string - Write(io.Writer) + fmt.Println("\n" + apiDiff) } -func diffAPI(version, prev string) (string, error) { +func diffAPI(oldVer, newVer string) (string, error) { ctx := context.Background() - previousApi, err := loadAPI(ctx, prev) + previousAPI, err := loadAPI(ctx, oldVer) if err != nil { - return "", fmt.Errorf("load previous API: %v", err) + return "", fmt.Errorf("loading %s: %v", oldVer, err) } - var currentApi *source.APIJSON - if version == "" { - currentApi = source.GeneratedAPIJSON + var currentAPI *source.APIJSON + if newVer == "" { + currentAPI = source.GeneratedAPIJSON } else { var err error - currentApi, err = loadAPI(ctx, version) + currentAPI, err = loadAPI(ctx, newVer) if err != nil { - return "", fmt.Errorf("load current API: %v", err) + return "", fmt.Errorf("loading %s: %v", newVer, err) } } - b := &strings.Builder{} - if err := diff(b, previousApi.Commands, currentApi.Commands, "command", func(c *source.CommandJSON) string { - return c.Command - }, diffCommands); err != nil { - return "", fmt.Errorf("diff commands: %v", err) - } - if diff(b, previousApi.Analyzers, currentApi.Analyzers, "analyzer", func(a *source.AnalyzerJSON) string { - return a.Name - }, diffAnalyzers); err != nil { - return "", fmt.Errorf("diff analyzers: %v", err) - } - if err := diff(b, previousApi.Lenses, currentApi.Lenses, "code lens", func(l *source.LensJSON) string { - return l.Lens - }, diffLenses); err != nil { - return "", fmt.Errorf("diff lenses: %v", err) - } - for key, prev := range previousApi.Options { - current, ok := currentApi.Options[key] - if !ok { - panic(fmt.Sprintf("unexpected option key: %s", key)) - } - if err := diff(b, prev, current, "option", func(o *source.OptionJSON) string { - return o.Name - }, diffOptions); err != nil { - return "", fmt.Errorf("diff options (%s): %v", key, err) - } - } - - return b.String(), nil + return cmp.Diff(previousAPI, currentAPI), nil } -func diff[T JSON](b *strings.Builder, previous, new []T, kind string, uniqueKey func(T) string, diffFunc func(*strings.Builder, T, T)) error { - prevJSON := collect(previous, uniqueKey) - newJSON := collect(new, uniqueKey) - for k := range newJSON { - delete(prevJSON, k) - } - for _, deleted := range prevJSON { - b.WriteString(fmt.Sprintf("%s %s was deleted.\n", kind, deleted)) - } - for _, prev := range previous { - delete(newJSON, uniqueKey(prev)) - } - if len(newJSON) > 0 { - b.WriteString("The following commands were added:\n") - for _, n := range newJSON { - n.Write(b) - b.WriteByte('\n') - } - } - previousMap := collect(previous, uniqueKey) - for _, current := range new { - prev, ok := previousMap[uniqueKey(current)] - if !ok { - continue - } - c, p := bytes.NewBuffer(nil), bytes.NewBuffer(nil) - prev.Write(p) - current.Write(c) - if diff := diffStr(p.String(), c.String()); diff != "" { - diffFunc(b, prev, current) - b.WriteString("\n--\n") - } - } - return nil -} - -func collect[T JSON](args []T, uniqueKey func(T) string) map[string]T { - m := map[string]T{} - for _, arg := range args { - m[uniqueKey(arg)] = arg - } - return m -} - -var goCmdRunner = gocommand.Runner{} - func loadAPI(ctx context.Context, version string) (*source.APIJSON, error) { - tmpGopath, err := ioutil.TempDir("", "gopath*") - if err != nil { - return nil, fmt.Errorf("temp dir: %v", err) - } - defer os.RemoveAll(tmpGopath) + ver := fmt.Sprintf("golang.org/x/tools/gopls@%s", version) + cmd := exec.Command("go", "run", ver, "api-json") - exampleDir := fmt.Sprintf("%s/src/example.com", tmpGopath) - if err := os.MkdirAll(exampleDir, 0776); err != nil { - return nil, fmt.Errorf("mkdir: %v", err) - } + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + cmd.Stdout = stdout + cmd.Stderr = stderr - if stdout, err := goCmdRunner.Run(ctx, gocommand.Invocation{ - Verb: "mod", - Args: []string{"init", "example.com"}, - WorkingDir: exampleDir, - Env: append(os.Environ(), fmt.Sprintf("GOPATH=%s", tmpGopath)), - }); err != nil { - return nil, fmt.Errorf("go mod init failed: %v (stdout: %v)", err, stdout) - } - if stdout, err := goCmdRunner.Run(ctx, gocommand.Invocation{ - Verb: "install", - Args: []string{fmt.Sprintf("golang.org/x/tools/gopls@%s", version)}, - WorkingDir: exampleDir, - Env: append(os.Environ(), fmt.Sprintf("GOPATH=%s", tmpGopath)), - }); err != nil { - return nil, fmt.Errorf("go install failed: %v (stdout: %v)", err, stdout.String()) - } - cmd := exec.Cmd{ - Path: filepath.Join(tmpGopath, "bin", "gopls"), - Args: []string{"gopls", "api-json"}, - Dir: tmpGopath, - } - out, err := cmd.Output() - if err != nil { - return nil, fmt.Errorf("output: %v", err) + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("go run failed: %v; stderr:\n%s", err, stderr) } apiJson := &source.APIJSON{} - if err := json.Unmarshal(out, apiJson); err != nil { + if err := json.Unmarshal(stdout.Bytes(), apiJson); err != nil { return nil, fmt.Errorf("unmarshal: %v", err) } return apiJson, nil } - -func diffCommands(b *strings.Builder, prev, current *source.CommandJSON) { - if prev.Title != current.Title { - b.WriteString(fmt.Sprintf("Title changed from %q to %q\n", prev.Title, current.Title)) - } - if prev.Doc != current.Doc { - b.WriteString(fmt.Sprintf("Documentation changed from %q to %q\n", prev.Doc, current.Doc)) - } - if prev.ArgDoc != current.ArgDoc { - b.WriteString("Arguments changed from " + formatBlock(prev.ArgDoc) + " to " + formatBlock(current.ArgDoc)) - } - if prev.ResultDoc != current.ResultDoc { - b.WriteString("Results changed from " + formatBlock(prev.ResultDoc) + " to " + formatBlock(current.ResultDoc)) - } -} - -func diffAnalyzers(b *strings.Builder, previous, current *source.AnalyzerJSON) { - b.WriteString(fmt.Sprintf("Changes to analyzer %s:\n\n", current.Name)) - if previous.Doc != current.Doc { - b.WriteString(fmt.Sprintf("Documentation changed from %q to %q\n", previous.Doc, current.Doc)) - } - if previous.Default != current.Default { - b.WriteString(fmt.Sprintf("Default changed from %v to %v\n", previous.Default, current.Default)) - } -} - -func diffLenses(b *strings.Builder, previous, current *source.LensJSON) { - b.WriteString(fmt.Sprintf("Changes to code lens %s:\n\n", current.Title)) - if previous.Title != current.Title { - b.WriteString(fmt.Sprintf("Title changed from %q to %q\n", previous.Title, current.Title)) - } - if previous.Doc != current.Doc { - b.WriteString(fmt.Sprintf("Documentation changed from %q to %q\n", previous.Doc, current.Doc)) - } -} - -func diffOptions(b *strings.Builder, previous, current *source.OptionJSON) { - b.WriteString(fmt.Sprintf("Changes to option %s:\n\n", current.Name)) - if previous.Doc != current.Doc { - diff := diffStr(previous.Doc, current.Doc) - fmt.Fprintf(b, "Documentation changed:\n%s\n", diff) - } - if previous.Default != current.Default { - b.WriteString(fmt.Sprintf("Default changed from %q to %q\n", previous.Default, current.Default)) - } - if previous.Hierarchy != current.Hierarchy { - b.WriteString(fmt.Sprintf("Categorization changed from %q to %q\n", previous.Hierarchy, current.Hierarchy)) - } - if previous.Status != current.Status { - b.WriteString(fmt.Sprintf("Status changed from %q to %q\n", previous.Status, current.Status)) - } - if previous.Type != current.Type { - b.WriteString(fmt.Sprintf("Type changed from %q to %q\n", previous.Type, current.Type)) - } - // TODO(rstambler): Handle possibility of same number but different keys/values. - if len(previous.EnumKeys.Keys) != len(current.EnumKeys.Keys) { - b.WriteString(fmt.Sprintf("Enum keys changed from\n%s\n to \n%s\n", previous.EnumKeys, current.EnumKeys)) - } - if len(previous.EnumValues) != len(current.EnumValues) { - b.WriteString(fmt.Sprintf("Enum values changed from\n%s\n to \n%s\n", previous.EnumValues, current.EnumValues)) - } -} - -func formatBlock(str string) string { - if str == "" { - return `""` - } - return "\n```\n" + str + "\n```\n" -} - -func diffStr(before, after string) string { - if before == after { - return "" - } - // Add newlines to avoid newline messages in diff. - unified := diffpkg.Unified("previous", "current", before+"\n", after+"\n") - return fmt.Sprintf("%q", unified) -} diff --git a/gopls/doc/analyzers.md b/gopls/doc/analyzers.md index dfe2a43c2b9..176c32f1ba4 100644 --- a/gopls/doc/analyzers.md +++ b/gopls/doc/analyzers.md @@ -223,10 +223,11 @@ literal inside the loop body. It checks for patterns where access to a loop variable is known to escape the current loop iteration: 1. a call to go or defer at the end of the loop body 2. a call to golang.org/x/sync/errgroup.Group.Go at the end of the loop body + 3. a call testing.T.Run where the subtest body invokes t.Parallel() -The analyzer only considers references in the last statement of the loop body -as it is not deep enough to understand the effects of subsequent statements -which might render the reference benign. +In the case of (1) and (2), the analyzer only considers references in the last +statement of the loop body as it is not deep enough to understand the effects +of subsequent statements which might render the reference benign. For example: diff --git a/gopls/doc/releases.md b/gopls/doc/releases.md new file mode 100644 index 00000000000..befb92c3966 --- /dev/null +++ b/gopls/doc/releases.md @@ -0,0 +1,25 @@ +# Gopls release policy + +Gopls releases follow [semver](http://semver.org), with major changes and new +features introduced only in new minor versions (i.e. versions of the form +`v*.N.0` for some N). Subsequent patch releases contain only cherry-picked +fixes or superficial updates. + +In order to align with the +[Go release timeline](https://github.com/golang/go/wiki/Go-Release-Cycle#timeline), +we aim to release a new minor version of Gopls approximately every three +months, with patch releases approximately every month, according to the +following table: + +| Month | Version(s) | +| ---- | ------- | +| Jan | `v*..0` | +| Jan-Mar | `v*..*` | +| Apr | `v*..0` | +| Apr-Jun | `v*..*` | +| Jul | `v*..0` | +| Jul-Sep | `v*..*` | +| Oct | `v*..0` | +| Oct-Dec | `v*..*` | + +For more background on this policy, see https://go.dev/issue/55267. diff --git a/gopls/doc/settings.md b/gopls/doc/settings.md index 34fb8d01ce8..5595976363f 100644 --- a/gopls/doc/settings.md +++ b/gopls/doc/settings.md @@ -510,13 +510,14 @@ Default: `false`. #### **newDiff** *string* -newDiff enables the new diff implementation. If this is "both", -for now both diffs will be run and statistics will be generateted in -a file in $TMPDIR. This is a risky setting; help in trying it -is appreciated. If it is "old" the old implementation is used, -and if it is "new", just the new implementation is used. - -Default: 'old'. +newDiff enables the new diff implementation. If this is "both", for now both +diffs will be run and statistics will be generated in a file in $TMPDIR. This +is a risky setting; help in trying it is appreciated. If it is "old" the old +implementation is used, and if it is "new", just the new implementation is +used. This setting will eventually be deleted, once gopls has fully migrated to +the new diff algorithm. + +Default: 'both'. ## Code Lenses diff --git a/gopls/doc/workspace.md b/gopls/doc/workspace.md index cb166131fba..4ff9994f939 100644 --- a/gopls/doc/workspace.md +++ b/gopls/doc/workspace.md @@ -34,12 +34,14 @@ your workspace root to the directory containing the `go.work` file. For example, suppose this repo is checked out into the `$WORK/tools` directory. We can work on both `golang.org/x/tools` and `golang.org/x/tools/gopls` -simultaneously by creating a `go.work` file: +simultaneously by creating a `go.work` file using `go work init`, followed by +`go work use MODULE_DIRECTORIES...` to add directories containing `go.mod` files to the +workspace: -``` +```sh cd $WORK go work init -go work use tools tools/gopls +go work use ./tools/ ./tools/gopls/ ``` ...followed by opening the `$WORK` directory in our editor. diff --git a/gopls/go.mod b/gopls/go.mod index efb9be1189e..979174c67c5 100644 --- a/gopls/go.mod +++ b/gopls/go.mod @@ -3,28 +3,28 @@ module golang.org/x/tools/gopls go 1.18 require ( - github.com/google/go-cmp v0.5.8 + github.com/google/go-cmp v0.5.9 github.com/jba/printsrc v0.2.2 github.com/jba/templatecheck v0.6.0 github.com/sergi/go-diff v1.1.0 - golang.org/x/mod v0.6.0 - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 - golang.org/x/sys v0.1.0 + golang.org/x/mod v0.7.0 + golang.org/x/sync v0.1.0 + golang.org/x/sys v0.2.0 golang.org/x/text v0.4.0 - golang.org/x/tools v0.1.13-0.20220928184430-f80e98464e27 + golang.org/x/tools v0.2.0 golang.org/x/vuln v0.0.0-20221010193109-563322be2ea9 gopkg.in/yaml.v3 v3.0.1 honnef.co/go/tools v0.3.3 - mvdan.cc/gofumpt v0.3.1 + mvdan.cc/gofumpt v0.4.0 mvdan.cc/xurls/v2 v2.4.0 ) -require golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect +require golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 // indirect require ( - github.com/BurntSushi/toml v1.2.0 // indirect - github.com/google/safehtml v0.0.2 // indirect - golang.org/x/exp/typeparams v0.0.0-20220722155223-a9213eeb770e // indirect + github.com/BurntSushi/toml v1.2.1 // indirect + github.com/google/safehtml v0.1.0 // indirect + golang.org/x/exp/typeparams v0.0.0-20221031165847-c99f073a8326 // indirect ) replace golang.org/x/tools => ../ diff --git a/gopls/go.sum b/gopls/go.sum index 78ff483dc7a..9810051995b 100644 --- a/gopls/go.sum +++ b/gopls/go.sum @@ -1,23 +1,25 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= -github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/frankban/quicktest v1.14.2 h1:SPb1KFFmM+ybpEjPUhCCkZOM5xlovT5UbrMvWnXyBns= -github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786/go.mod h1:apVn/GCasLZUVpAJ6oWAuyP7Ne7CEsQbTnc0plM3m+o= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/safehtml v0.0.2 h1:ZOt2VXg4x24bW0m2jtzAOkhoXV0iM8vNKc0paByCZqM= github.com/google/safehtml v0.0.2/go.mod h1:L4KWwDsUJdECRAEpZoBn3O64bQaywRscowZjJAzjHnU= +github.com/google/safehtml v0.1.0 h1:EwLKo8qawTKfsi0orxcQAZzu07cICaBeFMegAU9eaT8= +github.com/google/safehtml v0.1.0/go.mod h1:L4KWwDsUJdECRAEpZoBn3O64bQaywRscowZjJAzjHnU= github.com/jba/printsrc v0.2.2 h1:9OHK51UT+/iMAEBlQIIXW04qvKyF3/vvLuwW/hL8tDU= github.com/jba/printsrc v0.2.2/go.mod h1:1xULjw59sL0dPdWpDoVU06TIEO/Wnfv6AHRpiElTwYM= github.com/jba/templatecheck v0.6.0 h1:SwM8C4hlK/YNLsdcXStfnHWE2HKkuTVwy5FKQHt5ro8= @@ -33,8 +35,9 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -43,30 +46,37 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= +golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE= +golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= -golang.org/x/exp/typeparams v0.0.0-20220722155223-a9213eeb770e h1:7Xs2YCOpMlNqSQSmrrnhlzBXIE/bpMecZplbLePTJvE= -golang.org/x/exp/typeparams v0.0.0-20220722155223-a9213eeb770e/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/exp/typeparams v0.0.0-20221031165847-c99f073a8326 h1:fl8k2zg28yA23264d82M4dp+YlJ3ngDcpuB1bewkQi4= +golang.org/x/exp/typeparams v0.0.0-20221031165847-c99f073a8326/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211213223007-03aa0b5f6827/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= @@ -88,8 +98,8 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY= honnef.co/go/tools v0.3.3 h1:oDx7VAwstgpYpb3wv0oxiZlxY+foCpRAwY7Vk6XpAgA= honnef.co/go/tools v0.3.3/go.mod h1:jzwdWgg7Jdq75wlfblQxO4neNaFFSvgc1tD5Wv8U0Yw= -mvdan.cc/gofumpt v0.3.1 h1:avhhrOmv0IuvQVK7fvwV91oFSGAk5/6Po8GXTzICeu8= -mvdan.cc/gofumpt v0.3.1/go.mod h1:w3ymliuxvzVx8DAutBnVyDqYb1Niy/yCJt/lk821YCE= +mvdan.cc/gofumpt v0.4.0 h1:JVf4NN1mIpHogBj7ABpgOyZc65/UUOkKQFkoURsz4MM= +mvdan.cc/gofumpt v0.4.0/go.mod h1:PljLOHDeZqgS8opHRKLzp2It2VBuSdteAgqUfzMTxlQ= mvdan.cc/unparam v0.0.0-20211214103731-d0ef000c54e5 h1:Jh3LAeMt1eGpxomyu3jVkmVZWW2MxZ1qIIV2TZ/nRio= mvdan.cc/unparam v0.0.0-20211214103731-d0ef000c54e5/go.mod h1:b8RRCBm0eeiWR8cfN88xeq2G5SG3VKGO+5UPWi5FSOY= mvdan.cc/xurls/v2 v2.4.0 h1:tzxjVAj+wSBmDcF6zBB7/myTy3gX9xvi8Tyr28AuQgc= diff --git a/gopls/internal/coverage/coverage.go b/gopls/internal/coverage/coverage.go index 1ceabefab58..9a7d219945e 100644 --- a/gopls/internal/coverage/coverage.go +++ b/gopls/internal/coverage/coverage.go @@ -188,7 +188,12 @@ func maybePrint(m result) { if *verbose > 3 { fmt.Printf("%s %s %q %.3f\n", m.Action, m.Test, m.Output, m.Elapsed) } + case "pause", "cont": + if *verbose > 2 { + fmt.Printf("%s %s %.3f\n", m.Action, m.Test, m.Elapsed) + } default: + fmt.Printf("%#v\n", m) log.Fatalf("unknown action %s\n", m.Action) } } @@ -228,7 +233,7 @@ func checkCwd() { if err != nil { log.Fatal(err) } - // we expect to be a the root of golang.org/x/tools + // we expect to be at the root of golang.org/x/tools cmd := exec.Command("go", "list", "-m", "-f", "{{.Dir}}", "golang.org/x/tools") buf, err := cmd.Output() buf = bytes.Trim(buf, "\n \t") // remove \n at end @@ -243,10 +248,6 @@ func checkCwd() { if err != nil { log.Fatalf("expected a gopls directory, %v", err) } - _, err = os.Stat("internal/lsp") - if err != nil { - log.Fatalf("expected to see internal/lsp, %v", err) - } } func listDirs(dir string) []string { diff --git a/gopls/internal/hooks/analysis.go b/gopls/internal/hooks/analysis.go deleted file mode 100644 index 27ab9a699f9..00000000000 --- a/gopls/internal/hooks/analysis.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2019 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build go1.17 -// +build go1.17 - -package hooks - -import ( - "golang.org/x/tools/gopls/internal/lsp/protocol" - "golang.org/x/tools/gopls/internal/lsp/source" - "honnef.co/go/tools/analysis/lint" - "honnef.co/go/tools/quickfix" - "honnef.co/go/tools/simple" - "honnef.co/go/tools/staticcheck" - "honnef.co/go/tools/stylecheck" -) - -func updateAnalyzers(options *source.Options) { - options.StaticcheckSupported = true - - mapSeverity := func(severity lint.Severity) protocol.DiagnosticSeverity { - switch severity { - case lint.SeverityError: - return protocol.SeverityError - case lint.SeverityDeprecated: - // TODO(dh): in LSP, deprecated is a tag, not a severity. - // We'll want to support this once we enable SA5011. - return protocol.SeverityWarning - case lint.SeverityWarning: - return protocol.SeverityWarning - case lint.SeverityInfo: - return protocol.SeverityInformation - case lint.SeverityHint: - return protocol.SeverityHint - default: - return protocol.SeverityWarning - } - } - add := func(analyzers []*lint.Analyzer, skip map[string]struct{}) { - for _, a := range analyzers { - if _, ok := skip[a.Analyzer.Name]; ok { - continue - } - - enabled := !a.Doc.NonDefault - options.AddStaticcheckAnalyzer(a.Analyzer, enabled, mapSeverity(a.Doc.Severity)) - } - } - - add(simple.Analyzers, nil) - add(staticcheck.Analyzers, map[string]struct{}{ - // This check conflicts with the vet printf check (golang/go#34494). - "SA5009": {}, - // This check relies on facts from dependencies, which - // we don't currently compute. - "SA5011": {}, - }) - add(stylecheck.Analyzers, nil) - add(quickfix.Analyzers, nil) -} diff --git a/gopls/internal/hooks/analysis_116.go b/gopls/internal/hooks/analysis_116.go new file mode 100644 index 00000000000..dd429dea898 --- /dev/null +++ b/gopls/internal/hooks/analysis_116.go @@ -0,0 +1,14 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !go1.17 +// +build !go1.17 + +package hooks + +import "golang.org/x/tools/gopls/internal/lsp/source" + +func updateAnalyzers(options *source.Options) { + options.StaticcheckSupported = false +} diff --git a/gopls/internal/hooks/analysis_117.go b/gopls/internal/hooks/analysis_117.go index dd429dea898..27ab9a699f9 100644 --- a/gopls/internal/hooks/analysis_117.go +++ b/gopls/internal/hooks/analysis_117.go @@ -1,14 +1,62 @@ -// Copyright 2021 The Go Authors. All rights reserved. +// Copyright 2019 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !go1.17 -// +build !go1.17 +//go:build go1.17 +// +build go1.17 package hooks -import "golang.org/x/tools/gopls/internal/lsp/source" +import ( + "golang.org/x/tools/gopls/internal/lsp/protocol" + "golang.org/x/tools/gopls/internal/lsp/source" + "honnef.co/go/tools/analysis/lint" + "honnef.co/go/tools/quickfix" + "honnef.co/go/tools/simple" + "honnef.co/go/tools/staticcheck" + "honnef.co/go/tools/stylecheck" +) func updateAnalyzers(options *source.Options) { - options.StaticcheckSupported = false + options.StaticcheckSupported = true + + mapSeverity := func(severity lint.Severity) protocol.DiagnosticSeverity { + switch severity { + case lint.SeverityError: + return protocol.SeverityError + case lint.SeverityDeprecated: + // TODO(dh): in LSP, deprecated is a tag, not a severity. + // We'll want to support this once we enable SA5011. + return protocol.SeverityWarning + case lint.SeverityWarning: + return protocol.SeverityWarning + case lint.SeverityInfo: + return protocol.SeverityInformation + case lint.SeverityHint: + return protocol.SeverityHint + default: + return protocol.SeverityWarning + } + } + add := func(analyzers []*lint.Analyzer, skip map[string]struct{}) { + for _, a := range analyzers { + if _, ok := skip[a.Analyzer.Name]; ok { + continue + } + + enabled := !a.Doc.NonDefault + options.AddStaticcheckAnalyzer(a.Analyzer, enabled, mapSeverity(a.Doc.Severity)) + } + } + + add(simple.Analyzers, nil) + add(staticcheck.Analyzers, map[string]struct{}{ + // This check conflicts with the vet printf check (golang/go#34494). + "SA5009": {}, + // This check relies on facts from dependencies, which + // we don't currently compute. + "SA5011": {}, + }) + add(stylecheck.Analyzers, nil) + add(quickfix.Analyzers, nil) } diff --git a/gopls/internal/hooks/diff.go b/gopls/internal/hooks/diff.go index cac136ae192..a0383b87675 100644 --- a/gopls/internal/hooks/diff.go +++ b/gopls/internal/hooks/diff.go @@ -5,19 +5,15 @@ package hooks import ( - "crypto/rand" "encoding/json" "fmt" - "io" + "io/ioutil" "log" - "math/big" "os" "path/filepath" "runtime" - "strings" "sync" "time" - "unicode" "github.com/sergi/go-diff/diffmatchpatch" "golang.org/x/tools/internal/bug" @@ -36,108 +32,74 @@ type diffstat struct { } var ( - mu sync.Mutex // serializes writes and protects ignored - difffd io.Writer - ignored int // lots of the diff calls have 0 diffs -) + ignoredMu sync.Mutex + ignored int // counter of diff requests on equal strings -var fileonce sync.Once + diffStatsOnce sync.Once + diffStats *os.File // never closed +) +// save writes a JSON record of statistics about diff requests to a temporary file. func (s *diffstat) save() { - // save log records in a file in os.TempDir(). - // diff is frequently called with identical strings, so - // these are somewhat compressed out - fileonce.Do(func() { - fname := filepath.Join(os.TempDir(), fmt.Sprintf("gopls-diff-%x", os.Getpid())) - fd, err := os.Create(fname) + diffStatsOnce.Do(func() { + f, err := ioutil.TempFile("", "gopls-diff-stats-*") if err != nil { - // now what? + log.Printf("can't create diff stats temp file: %v", err) // e.g. disk full + return } - difffd = fd + diffStats = f }) + if diffStats == nil { + return + } - mu.Lock() - defer mu.Unlock() + // diff is frequently called with equal strings, + // so we count repeated instances but only print every 15th. + ignoredMu.Lock() if s.Oldedits == 0 && s.Newedits == 0 { + ignored++ if ignored < 15 { - // keep track of repeated instances of no diffs - // but only print every 15th - ignored++ + ignoredMu.Unlock() return } - s.Ignored = ignored + 1 - } else { - s.Ignored = ignored } + s.Ignored = ignored ignored = 0 - // it would be really nice to see why diff was called - _, f, l, ok := runtime.Caller(2) - if ok { - var fname string - fname = filepath.Base(f) // diff is only called from a few places - s.Stack = fmt.Sprintf("%s:%d", fname, l) + ignoredMu.Unlock() + + // Record the name of the file in which diff was called. + // There aren't many calls, so only the base name is needed. + if _, file, line, ok := runtime.Caller(2); ok { + s.Stack = fmt.Sprintf("%s:%d", filepath.Base(file), line) } x, err := json.Marshal(s) if err != nil { - log.Print(err) // failure to print statistics should not stop gopls + log.Fatalf("internal error marshalling JSON: %v", err) } - fmt.Fprintf(difffd, "%s\n", x) + fmt.Fprintf(diffStats, "%s\n", x) } -// save encrypted versions of the broken input and return the file name -// (the saved strings will have the same diff behavior as the user's strings) +// disaster is called when the diff algorithm panics or produces a +// diff that cannot be applied. It saves the broken input in a +// new temporary file and logs the file name, which is returned. func disaster(before, after string) string { - // encrypt before and after for privacy. (randomized monoalphabetic cipher) - // got will contain the substitution cipher - // for the runes in before and after - got := map[rune]rune{} - for _, r := range before { - got[r] = ' ' // value doesn't matter - } - for _, r := range after { - got[r] = ' ' - } - repl := initrepl(len(got)) - i := 0 - for k := range got { // randomized - got[k] = repl[i] - i++ - } - // use got to encrypt before and after - subst := func(r rune) rune { return got[r] } - first := strings.Map(subst, before) - second := strings.Map(subst, after) - - // one failure per session is enough, and more private. - // this saves the last one. - fname := fmt.Sprintf("%s/gopls-failed-%x", os.TempDir(), os.Getpid()) - fd, err := os.Create(fname) - defer fd.Close() - _, err = fmt.Fprintf(fd, "%s\n%s\n", first, second) - if err != nil { - // what do we tell the user? + // We use the pid to salt the name, not os.TempFile, + // so that each process creates at most one file. + // One is sufficient for a bug report. + filename := fmt.Sprintf("%s/gopls-diff-bug-%x", os.TempDir(), os.Getpid()) + + // We use NUL as a separator: it should never appear in Go source. + data := before + "\x00" + after + + if err := ioutil.WriteFile(filename, []byte(data), 0600); err != nil { + log.Printf("failed to write diff bug report: %v", err) return "" } - // ask the user to send us the file, somehow - return fname -} -func initrepl(n int) []rune { - repl := make([]rune, 0, n) - for r := rune(0); len(repl) < n; r++ { - if unicode.IsLetter(r) || unicode.IsNumber(r) { - repl = append(repl, r) - } - } - // randomize repl - rdr := rand.Reader - lim := big.NewInt(int64(len(repl))) - for i := 1; i < n; i++ { - v, _ := rand.Int(rdr, lim) - k := v.Int64() - repl[i], repl[k] = repl[k], repl[i] - } - return repl + // TODO(adonovan): is there a better way to surface this? + log.Printf("Bug detected in diff algorithm! Please send file %s to the maintainers of gopls if you are comfortable sharing its contents.", filename) + + return filename } // BothDiffs edits calls both the new and old diffs, checks that the new diffs @@ -145,7 +107,7 @@ func initrepl(n int) []rune { func BothDiffs(before, after string) (edits []diff.Edit) { // The new diff code contains a lot of internal checks that panic when they // fail. This code catches the panics, or other failures, tries to save - // the failing example (and ut wiykd ask the user to send it back to us, and + // the failing example (and it would ask the user to send it back to us, and // changes options.newDiff to 'old', if only we could figure out how.) stat := diffstat{Before: len(before), After: len(after)} now := time.Now() diff --git a/gopls/internal/hooks/diff_test.go b/gopls/internal/hooks/diff_test.go index acc5d29991f..a46bf3b2d28 100644 --- a/gopls/internal/hooks/diff_test.go +++ b/gopls/internal/hooks/diff_test.go @@ -5,11 +5,9 @@ package hooks import ( - "fmt" "io/ioutil" "os" "testing" - "unicode/utf8" "golang.org/x/tools/internal/diff/difftest" ) @@ -18,40 +16,18 @@ func TestDiff(t *testing.T) { difftest.DiffTest(t, ComputeEdits) } -func TestRepl(t *testing.T) { - t.Skip("just for checking repl by looking at it") - repl := initrepl(800) - t.Errorf("%q", string(repl)) - t.Errorf("%d", len(repl)) -} - func TestDisaster(t *testing.T) { - a := "This is a string,(\u0995) just for basic functionality" - b := "Ths is another string, (\u0996) to see if disaster will store stuff correctly" + a := "This is a string,(\u0995) just for basic\nfunctionality" + b := "This is another string, (\u0996) to see if disaster will store stuff correctly" fname := disaster(a, b) buf, err := ioutil.ReadFile(fname) if err != nil { - t.Errorf("error %v reading %s", err, fname) - } - var x, y string - n, err := fmt.Sscanf(string(buf), "%s\n%s\n", &x, &y) - if n != 2 { - t.Errorf("got %d, expected 2", n) - t.Logf("read %q", string(buf)) - } - if a == x || b == y { - t.Error("failed to encrypt") - } - err = os.Remove(fname) - if err != nil { - t.Errorf("%v removing %s", err, fname) + t.Fatal(err) } - alen, blen := utf8.RuneCount([]byte(a)), utf8.RuneCount([]byte(b)) - xlen, ylen := utf8.RuneCount([]byte(x)), utf8.RuneCount([]byte(y)) - if alen != xlen { - t.Errorf("a; got %d, expected %d", xlen, alen) + if string(buf) != a+"\x00"+b { + t.Error("failed to record original strings") } - if blen != ylen { - t.Errorf("b: got %d expected %d", ylen, blen) + if err := os.Remove(fname); err != nil { + t.Error(err) } } diff --git a/gopls/internal/hooks/gofumpt_117.go b/gopls/internal/hooks/gofumpt_117.go new file mode 100644 index 00000000000..71886357704 --- /dev/null +++ b/gopls/internal/hooks/gofumpt_117.go @@ -0,0 +1,13 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !go1.18 +// +build !go1.18 + +package hooks + +import "golang.org/x/tools/gopls/internal/lsp/source" + +func updateGofumpt(options *source.Options) { +} diff --git a/gopls/internal/hooks/gofumpt_118.go b/gopls/internal/hooks/gofumpt_118.go new file mode 100644 index 00000000000..4eb523261dc --- /dev/null +++ b/gopls/internal/hooks/gofumpt_118.go @@ -0,0 +1,24 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.18 +// +build go1.18 + +package hooks + +import ( + "context" + + "golang.org/x/tools/gopls/internal/lsp/source" + "mvdan.cc/gofumpt/format" +) + +func updateGofumpt(options *source.Options) { + options.GofumptFormat = func(ctx context.Context, langVersion, modulePath string, src []byte) ([]byte, error) { + return format.Source(src, format.Options{ + LangVersion: langVersion, + ModulePath: modulePath, + }) + } +} diff --git a/gopls/internal/hooks/hooks.go b/gopls/internal/hooks/hooks.go index 085fa53a39a..5624a5eb386 100644 --- a/gopls/internal/hooks/hooks.go +++ b/gopls/internal/hooks/hooks.go @@ -8,11 +8,8 @@ package hooks // import "golang.org/x/tools/gopls/internal/hooks" import ( - "context" - "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/internal/diff" - "mvdan.cc/gofumpt/format" "mvdan.cc/xurls/v2" ) @@ -29,11 +26,6 @@ func Options(options *source.Options) { } } options.URLRegexp = xurls.Relaxed() - options.GofumptFormat = func(ctx context.Context, langVersion, modulePath string, src []byte) ([]byte, error) { - return format.Source(src, format.Options{ - LangVersion: langVersion, - ModulePath: modulePath, - }) - } updateAnalyzers(options) + updateGofumpt(options) } diff --git a/gopls/internal/hooks/licenses_test.go b/gopls/internal/hooks/licenses_test.go index 3b61d348d95..b10d7e2b36c 100644 --- a/gopls/internal/hooks/licenses_test.go +++ b/gopls/internal/hooks/licenses_test.go @@ -15,9 +15,9 @@ import ( ) func TestLicenses(t *testing.T) { - // License text differs for older Go versions because staticcheck isn't - // supported for those versions. - testenv.NeedsGo1Point(t, 17) + // License text differs for older Go versions because staticcheck or gofumpt + // isn't supported for those versions. + testenv.NeedsGo1Point(t, 18) if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { t.Skip("generating licenses only works on Unixes") diff --git a/gopls/internal/lsp/cache/analysis.go b/gopls/internal/lsp/cache/analysis.go index fa2f7eaf08f..e15b43e91ce 100644 --- a/gopls/internal/lsp/cache/analysis.go +++ b/gopls/internal/lsp/cache/analysis.go @@ -150,8 +150,8 @@ func (s *snapshot) actionHandle(ctx context.Context, id PackageID, a *analysis.A // An analysis that consumes/produces facts // must run on the package's dependencies too. if len(a.FactTypes) > 0 { - for _, importID := range ph.m.Imports { - depActionHandle, err := s.actionHandle(ctx, importID, a) + for _, depID := range ph.m.DepsByPkgPath { + depActionHandle, err := s.actionHandle(ctx, depID, a) if err != nil { return nil, err } diff --git a/gopls/internal/lsp/cache/check.go b/gopls/internal/lsp/cache/check.go index 70eaed0d193..17943efb707 100644 --- a/gopls/internal/lsp/cache/check.go +++ b/gopls/internal/lsp/cache/check.go @@ -100,11 +100,7 @@ func (s *snapshot) buildPackageHandle(ctx context.Context, id PackageID, mode so // the recursive key building of dependencies in parallel. deps := make(map[PackageID]*packageHandle) var depKey source.Hash // XOR of all unique deps - for _, depID := range m.Imports { - depHandle, ok := deps[depID] - if ok { - continue // e.g. duplicate import - } + for _, depID := range m.DepsByPkgPath { depHandle, err := s.buildPackageHandle(ctx, depID, s.workspaceParseMode(depID)) // Don't use invalid metadata for dependencies if the top-level // metadata is valid. We only load top-level packages, so if the @@ -452,10 +448,10 @@ func doTypeCheck(ctx context.Context, snapshot *snapshot, goFiles, compiledGoFil defer done() pkg := &pkg{ - m: m, - mode: mode, - depsByPkgPath: make(map[PackagePath]*pkg), - types: types.NewPackage(string(m.PkgPath), string(m.Name)), + m: m, + mode: mode, + deps: make(map[PackageID]*pkg), + types: types.NewPackage(string(m.PkgPath), string(m.Name)), typesInfo: &types.Info{ Types: make(map[ast.Expr]types.TypeAndValue), Defs: make(map[*ast.Ident]types.Object), @@ -528,16 +524,16 @@ func doTypeCheck(ctx context.Context, snapshot *snapshot, goFiles, compiledGoFil // based on the metadata before we start type checking, // reporting them via types.Importer places the errors // at the correct source location. - id, ok := pkg.m.Imports[ImportPath(path)] + id, ok := pkg.m.DepsByImpPath[ImportPath(path)] if !ok { // If the import declaration is broken, // go list may fail to report metadata about it. // See TestFixImportDecl for an example. return nil, fmt.Errorf("missing metadata for import of %q", path) } - dep, ok := deps[id] + dep, ok := deps[id] // id may be "" if !ok { - return nil, snapshot.missingPkgError(ctx, path) + return nil, snapshot.missingPkgError(path) } if !source.IsValidImport(string(m.PkgPath), string(dep.m.PkgPath)) { return nil, fmt.Errorf("invalid use of internal package %s", path) @@ -546,7 +542,7 @@ func doTypeCheck(ctx context.Context, snapshot *snapshot, goFiles, compiledGoFil if err != nil { return nil, err } - pkg.depsByPkgPath[depPkg.m.PkgPath] = depPkg + pkg.deps[depPkg.m.ID] = depPkg return depPkg.types, nil }), } @@ -761,21 +757,19 @@ func (s *snapshot) depsErrors(ctx context.Context, pkg *pkg) ([]*source.Diagnost // missingPkgError returns an error message for a missing package that varies // based on the user's workspace mode. -func (s *snapshot) missingPkgError(ctx context.Context, pkgPath string) error { +func (s *snapshot) missingPkgError(pkgPath string) error { var b strings.Builder if s.workspaceMode()&moduleMode == 0 { gorootSrcPkg := filepath.FromSlash(filepath.Join(s.view.goroot, "src", pkgPath)) - - b.WriteString(fmt.Sprintf("cannot find package %q in any of \n\t%s (from $GOROOT)", pkgPath, gorootSrcPkg)) - + fmt.Fprintf(&b, "cannot find package %q in any of \n\t%s (from $GOROOT)", pkgPath, gorootSrcPkg) for _, gopath := range filepath.SplitList(s.view.gopath) { gopathSrcPkg := filepath.FromSlash(filepath.Join(gopath, "src", pkgPath)) - b.WriteString(fmt.Sprintf("\n\t%s (from $GOPATH)", gopathSrcPkg)) + fmt.Fprintf(&b, "\n\t%s (from $GOPATH)", gopathSrcPkg) } } else { - b.WriteString(fmt.Sprintf("no required module provides package %q", pkgPath)) - if err := s.getInitializationError(ctx); err != nil { - b.WriteString(fmt.Sprintf("(workspace configuration error: %s)", err.MainError)) + fmt.Fprintf(&b, "no required module provides package %q", pkgPath) + if err := s.getInitializationError(); err != nil { + fmt.Fprintf(&b, "\n(workspace configuration error: %s)", err.MainError) } } return errors.New(b.String()) diff --git a/gopls/internal/lsp/cache/graph.go b/gopls/internal/lsp/cache/graph.go index 047a55c7920..1dac0767afb 100644 --- a/gopls/internal/lsp/cache/graph.go +++ b/gopls/internal/lsp/cache/graph.go @@ -53,8 +53,8 @@ func (g *metadataGraph) build() { // Build the import graph. g.importedBy = make(map[PackageID][]PackageID) for id, m := range g.metadata { - for _, importID := range uniqueDeps(m.Imports) { - g.importedBy[importID] = append(g.importedBy[importID], id) + for _, depID := range m.DepsByPkgPath { + g.importedBy[depID] = append(g.importedBy[depID], id) } } @@ -129,25 +129,6 @@ func (g *metadataGraph) build() { } } -// uniqueDeps returns a new sorted and duplicate-free slice containing the -// IDs of the package's direct dependencies. -func uniqueDeps(imports map[ImportPath]PackageID) []PackageID { - // TODO(adonovan): use generic maps.SortedUniqueValues(m.Imports) when available. - ids := make([]PackageID, 0, len(imports)) - for _, id := range imports { - ids = append(ids, id) - } - sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) - // de-duplicate in place - out := ids[:0] - for _, id := range ids { - if len(out) == 0 || id != out[len(out)-1] { - out = append(out, id) - } - } - return out -} - // reverseTransitiveClosure calculates the set of packages that transitively // import an id in ids. The result also includes given ids. // diff --git a/gopls/internal/lsp/cache/imports.go b/gopls/internal/lsp/cache/imports.go index a73f9beff3e..2bda377746d 100644 --- a/gopls/internal/lsp/cache/imports.go +++ b/gopls/internal/lsp/cache/imports.go @@ -13,11 +13,11 @@ import ( "sync" "time" + "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/internal/event" "golang.org/x/tools/internal/event/keys" "golang.org/x/tools/internal/gocommand" "golang.org/x/tools/internal/imports" - "golang.org/x/tools/gopls/internal/lsp/source" ) type importsState struct { @@ -37,33 +37,18 @@ func (s *importsState) runProcessEnvFunc(ctx context.Context, snapshot *snapshot s.mu.Lock() defer s.mu.Unlock() - // Find the hash of the active mod file, if any. Using the unsaved content + // Find the hash of active mod files, if any. Using the unsaved content // is slightly wasteful, since we'll drop caches a little too often, but // the mod file shouldn't be changing while people are autocompleting. - var modFileHash source.Hash - // If we are using 'legacyWorkspace' mode, we can just read the modfile from - // the snapshot. Otherwise, we need to get the synthetic workspace mod file. // - // TODO(rfindley): we should be able to just always use the synthetic - // workspace module, or alternatively use the go.work file. - if snapshot.workspace.moduleSource == legacyWorkspace { - for m := range snapshot.workspace.getActiveModFiles() { // range to access the only element - modFH, err := snapshot.GetFile(ctx, m) - if err != nil { - return err - } - modFileHash = modFH.FileIdentity().Hash - } - } else { - modFile, err := snapshot.workspace.modFile(ctx, snapshot) - if err != nil { - return err - } - modBytes, err := modFile.Format() + // TODO(rfindley): consider instead hashing on-disk modfiles here. + var modFileHash source.Hash + for m := range snapshot.workspace.ActiveModFiles() { + fh, err := snapshot.GetFile(ctx, m) if err != nil { return err } - modFileHash = source.HashOf(modBytes) + modFileHash.XORWith(fh.FileIdentity().Hash) } // view.goEnv is immutable -- changes make a new view. Options can change. diff --git a/gopls/internal/lsp/cache/load.go b/gopls/internal/lsp/cache/load.go index 83ea2cab41b..9cabffc0d03 100644 --- a/gopls/internal/lsp/cache/load.go +++ b/gopls/internal/lsp/cache/load.go @@ -352,10 +352,10 @@ https://github.com/golang/tools/blob/master/gopls/doc/workspace.md.` // If the user has one active go.mod file, they may still be editing files // in nested modules. Check the module of each open file and add warnings // that the nested module must be opened as a workspace folder. - if len(s.workspace.getActiveModFiles()) == 1 { + if len(s.workspace.ActiveModFiles()) == 1 { // Get the active root go.mod file to compare against. var rootModURI span.URI - for uri := range s.workspace.getActiveModFiles() { + for uri := range s.workspace.ActiveModFiles() { rootModURI = uri } nestedModules := map[string][]source.VersionedFileHandle{} @@ -542,10 +542,10 @@ func buildMetadata(ctx context.Context, pkg *packages.Package, cfg *packages.Con m.GoFiles = append(m.GoFiles, uri) } - imports := make(map[ImportPath]PackageID) + depsByImpPath := make(map[ImportPath]PackageID) + depsByPkgPath := make(map[PackagePath]PackageID) for importPath, imported := range pkg.Imports { importPath := ImportPath(importPath) - imports[importPath] = PackageID(imported.ID) // It is not an invariant that importPath == imported.PkgPath. // For example, package "net" imports "golang.org/x/net/dns/dnsmessage" @@ -590,18 +590,18 @@ func buildMetadata(ctx context.Context, pkg *packages.Package, cfg *packages.Con // TODO(adonovan): clarify this. Perhaps go/packages should // report which nodes were synthesized. if importPath != "unsafe" && len(imported.CompiledGoFiles) == 0 { - if m.MissingDeps == nil { - m.MissingDeps = make(map[ImportPath]struct{}) - } - m.MissingDeps[importPath] = struct{}{} + depsByImpPath[importPath] = "" // missing continue } + depsByImpPath[importPath] = PackageID(imported.ID) + depsByPkgPath[PackagePath(imported.PkgPath)] = PackageID(imported.ID) if err := buildMetadata(ctx, imported, cfg, query, updates, append(path, id)); err != nil { event.Error(ctx, "error in dependency", err) } } - m.Imports = imports + m.DepsByImpPath = depsByImpPath + m.DepsByPkgPath = depsByPkgPath return nil } diff --git a/gopls/internal/lsp/cache/metadata.go b/gopls/internal/lsp/cache/metadata.go index 5ac741a338f..c8b0a537222 100644 --- a/gopls/internal/lsp/cache/metadata.go +++ b/gopls/internal/lsp/cache/metadata.go @@ -33,8 +33,8 @@ type Metadata struct { ForTest PackagePath // package path under test, or "" TypesSizes types.Sizes Errors []packages.Error - Imports map[ImportPath]PackageID // may contain duplicate IDs - MissingDeps map[ImportPath]struct{} + DepsByImpPath map[ImportPath]PackageID // may contain dups; empty ID => missing + DepsByPkgPath map[PackagePath]PackageID // values are unique and non-empty Module *packages.Module depsErrors []*packagesinternal.PackageError diff --git a/gopls/internal/lsp/cache/pkg.go b/gopls/internal/lsp/cache/pkg.go index 2f7389db18e..ddfb9ea7e96 100644 --- a/gopls/internal/lsp/cache/pkg.go +++ b/gopls/internal/lsp/cache/pkg.go @@ -9,6 +9,7 @@ import ( "go/ast" "go/scanner" "go/types" + "sort" "golang.org/x/mod/module" "golang.org/x/tools/gopls/internal/lsp/source" @@ -23,7 +24,7 @@ type pkg struct { goFiles []*source.ParsedGoFile compiledGoFiles []*source.ParsedGoFile diagnostics []*source.Diagnostic - depsByPkgPath map[PackagePath]*pkg + deps map[PackageID]*pkg // use m.DepsBy{Pkg,Imp}Path to look up ID version *module.Version parseErrors []scanner.ErrorList typeErrors []types.Error @@ -35,6 +36,8 @@ type pkg struct { analyses memoize.Store // maps analyzer.Name to Promise[actionResult] } +func (p *pkg) String() string { return p.ID() } + // A loadScope defines a package loading scope for use with go/packages. type loadScope interface { aScope() @@ -116,38 +119,28 @@ func (p *pkg) ForTest() string { // from an import declaration, use ResolveImportPath instead. // They may differ in case of vendoring.) func (p *pkg) DirectDep(pkgPath string) (source.Package, error) { - if imp := p.depsByPkgPath[PackagePath(pkgPath)]; imp != nil { - return imp, nil + if id, ok := p.m.DepsByPkgPath[PackagePath(pkgPath)]; ok { + if imp := p.deps[id]; imp != nil { + return imp, nil + } } - // Don't return a nil pointer because that still satisfies the interface. - return nil, fmt.Errorf("no imported package for %s", pkgPath) + return nil, fmt.Errorf("package does not import package with path %s", pkgPath) } // ResolveImportPath returns the directly imported dependency of this package, // given its ImportPath. See also DirectDep. func (p *pkg) ResolveImportPath(importPath string) (source.Package, error) { - if id, ok := p.m.Imports[ImportPath(importPath)]; ok { - for _, imported := range p.depsByPkgPath { - if PackageID(imported.ID()) == id { - return imported, nil - } + if id, ok := p.m.DepsByImpPath[ImportPath(importPath)]; ok && id != "" { + if imp := p.deps[id]; imp != nil { + return imp, nil } } return nil, fmt.Errorf("package does not import %s", importPath) } func (p *pkg) MissingDependencies() []string { - // We don't invalidate metadata for import deletions, so check the package - // imports via the *types.Package. Only use metadata if p.types is nil. - if p.types == nil { - var md []string - for importPath := range p.m.MissingDeps { - md = append(md, string(importPath)) - } - return md - } - - // This looks wrong. + // We don't invalidate metadata for import deletions, + // so check the package imports via the *types.Package. // // rfindley says: it looks like this is intending to implement // a heuristic "if go list couldn't resolve import paths to @@ -158,20 +151,31 @@ func (p *pkg) MissingDependencies() []string { // doesn't need that dep anymore we shouldn't show the warning". // But either we're outside of GOPATH/Module, or we're not... // - // TODO(adonovan): figure out what it is trying to do. - var md []string + // adonovan says: I think this effectively reverses the + // heuristic used by the type checker when Importer.Import + // returns an error---go/types synthesizes a package whose + // Path is the import path (sans "vendor/")---hence the + // dubious ImportPath() conversion. A blank DepsByImpPath + // entry means a missing import. + // + // If we invalidate the metadata for import deletions (which + // should be fast) then we can simply return the blank entries + // in DepsByImpPath. (They are PackageIDs not PackagePaths, + // but the caller only cares whether the set is empty!) + var missing []string for _, pkg := range p.types.Imports() { - if _, ok := p.m.MissingDeps[ImportPath(pkg.Path())]; ok { - md = append(md, pkg.Path()) + if id, ok := p.m.DepsByImpPath[ImportPath(pkg.Path())]; ok && id == "" { + missing = append(missing, pkg.Path()) } } - return md + sort.Strings(missing) + return missing } func (p *pkg) Imports() []source.Package { - var result []source.Package - for _, imp := range p.depsByPkgPath { - result = append(result, imp) + var result []source.Package // unordered + for _, dep := range p.deps { + result = append(result, dep) } return result } diff --git a/gopls/internal/lsp/cache/snapshot.go b/gopls/internal/lsp/cache/snapshot.go index c7e5e8ad640..b05f401c52a 100644 --- a/gopls/internal/lsp/cache/snapshot.go +++ b/gopls/internal/lsp/cache/snapshot.go @@ -250,7 +250,7 @@ func (s *snapshot) FileSet() *token.FileSet { func (s *snapshot) ModFiles() []span.URI { var uris []span.URI - for modURI := range s.workspace.getActiveModFiles() { + for modURI := range s.workspace.ActiveModFiles() { uris = append(uris, modURI) } return uris @@ -281,7 +281,7 @@ func (s *snapshot) ValidBuildConfiguration() bool { } // Check if the user is working within a module or if we have found // multiple modules in the workspace. - if len(s.workspace.getActiveModFiles()) > 0 { + if len(s.workspace.ActiveModFiles()) > 0 { return true } // The user may have a multiple directories in their GOPATH. @@ -308,7 +308,7 @@ func (s *snapshot) workspaceMode() workspaceMode { // If the view is not in a module and contains no modules, but still has a // valid workspace configuration, do not create the workspace module. // It could be using GOPATH or a different build system entirely. - if len(s.workspace.getActiveModFiles()) == 0 && validBuildConfiguration { + if len(s.workspace.ActiveModFiles()) == 0 && validBuildConfiguration { return mode } mode |= moduleMode @@ -480,7 +480,7 @@ func (s *snapshot) goCommandInvocation(ctx context.Context, flags source.Invocat if mode == source.LoadWorkspace { switch s.workspace.moduleSource { case legacyWorkspace: - for m := range s.workspace.getActiveModFiles() { // range to access the only element + for m := range s.workspace.ActiveModFiles() { // range to access the only element modURI = m } case goWorkWorkspace: @@ -892,7 +892,7 @@ func (s *snapshot) isActiveLocked(id PackageID) (active bool) { } // TODO(rfindley): it looks incorrect that we don't also check GoFiles here. // If a CGo file is open, we want to consider the package active. - for _, dep := range m.Imports { + for _, dep := range m.DepsByPkgPath { if s.isActiveLocked(dep) { return true } @@ -1231,14 +1231,15 @@ func (s *snapshot) CachedImportPaths(ctx context.Context) (map[string]source.Pac if err != nil { return } - for pkgPath, newPkg := range cachedPkg.depsByPkgPath { - if oldPkg, ok := results[string(pkgPath)]; ok { + for _, newPkg := range cachedPkg.deps { + pkgPath := newPkg.PkgPath() + if oldPkg, ok := results[pkgPath]; ok { // Using the same trick as NarrowestPackage, prefer non-variants. if len(newPkg.compiledGoFiles) < len(oldPkg.(*pkg).compiledGoFiles) { - results[string(pkgPath)] = newPkg + results[pkgPath] = newPkg } } else { - results[string(pkgPath)] = newPkg + results[pkgPath] = newPkg } } }) @@ -1527,7 +1528,7 @@ func (s *snapshot) awaitLoadedAllErrors(ctx context.Context) *source.CriticalErr return nil } -func (s *snapshot) getInitializationError(ctx context.Context) *source.CriticalError { +func (s *snapshot) getInitializationError() *source.CriticalError { s.mu.Lock() defer s.mu.Unlock() @@ -1893,8 +1894,11 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC // package with missing dependencies. if anyFileAdded { for id, metadata := range s.meta.metadata { - if len(metadata.MissingDeps) > 0 { - directIDs[id] = true + for _, impID := range metadata.DepsByImpPath { + if impID == "" { // missing import + directIDs[id] = true + break + } } } } diff --git a/gopls/internal/lsp/cache/view.go b/gopls/internal/lsp/cache/view.go index dec2cb0808b..a408ee7a03a 100644 --- a/gopls/internal/lsp/cache/view.go +++ b/gopls/internal/lsp/cache/view.go @@ -581,13 +581,13 @@ func (v *View) Session() *Session { func (s *snapshot) IgnoredFile(uri span.URI) bool { filename := uri.Filename() var prefixes []string - if len(s.workspace.getActiveModFiles()) == 0 { + if len(s.workspace.ActiveModFiles()) == 0 { for _, entry := range filepath.SplitList(s.view.gopath) { prefixes = append(prefixes, filepath.Join(entry, "src")) } } else { prefixes = append(prefixes, s.view.gomodcache) - for m := range s.workspace.getActiveModFiles() { + for m := range s.workspace.ActiveModFiles() { prefixes = append(prefixes, dirURI(m).Filename()) } } @@ -679,8 +679,8 @@ func (s *snapshot) loadWorkspace(ctx context.Context, firstAttempt bool) { }) } - if len(s.workspace.getActiveModFiles()) > 0 { - for modURI := range s.workspace.getActiveModFiles() { + if len(s.workspace.ActiveModFiles()) > 0 { + for modURI := range s.workspace.ActiveModFiles() { // Be careful not to add context cancellation errors as critical module // errors. fh, err := s.GetFile(ctx, modURI) diff --git a/gopls/internal/lsp/cache/workspace.go b/gopls/internal/lsp/cache/workspace.go index 2020718ea7d..b280ef369a8 100644 --- a/gopls/internal/lsp/cache/workspace.go +++ b/gopls/internal/lsp/cache/workspace.go @@ -73,6 +73,7 @@ type workspaceCommon struct { type workspace struct { workspaceCommon + // The source of modules in this workspace. moduleSource workspaceSource // activeModFiles holds the active go.mod files. @@ -192,11 +193,13 @@ func (ws *workspace) loadExplicitWorkspaceFile(ctx context.Context, fs source.Fi var noHardcodedWorkspace = errors.New("no hardcoded workspace") +// TODO(rfindley): eliminate getKnownModFiles. func (w *workspace) getKnownModFiles() map[span.URI]struct{} { return w.knownModFiles } -func (w *workspace) getActiveModFiles() map[span.URI]struct{} { +// ActiveModFiles returns the set of active mod files for the current workspace. +func (w *workspace) ActiveModFiles() map[span.URI]struct{} { return w.activeModFiles } diff --git a/gopls/internal/lsp/cache/workspace_test.go b/gopls/internal/lsp/cache/workspace_test.go index 37e8f2cc46d..188869562c5 100644 --- a/gopls/internal/lsp/cache/workspace_test.go +++ b/gopls/internal/lsp/cache/workspace_test.go @@ -386,7 +386,7 @@ func checkState(ctx context.Context, t *testing.T, fs source.FileSource, rel fak t.Errorf("module source = %v, want %v", got.moduleSource, want.source) } modules := make(map[span.URI]struct{}) - for k := range got.getActiveModFiles() { + for k := range got.ActiveModFiles() { modules[k] = struct{}{} } for _, modPath := range want.modules { diff --git a/gopls/internal/lsp/cmd/call_hierarchy.go b/gopls/internal/lsp/cmd/call_hierarchy.go index 7736eecbe59..295dea8b0d4 100644 --- a/gopls/internal/lsp/cmd/call_hierarchy.go +++ b/gopls/internal/lsp/cmd/call_hierarchy.go @@ -47,7 +47,7 @@ func (c *callHierarchy) Run(ctx context.Context, args ...string) error { defer conn.terminate(ctx) from := span.Parse(args[0]) - file := conn.AddFile(ctx, from.URI()) + file := conn.openFile(ctx, from.URI()) if file.err != nil { return file.err } @@ -114,7 +114,7 @@ func (c *callHierarchy) Run(ctx context.Context, args ...string) error { // callItemPrintString returns a protocol.CallHierarchyItem object represented as a string. // item and call ranges (protocol.Range) are converted to user friendly spans (1-indexed). func callItemPrintString(ctx context.Context, conn *connection, item protocol.CallHierarchyItem, callsURI protocol.DocumentURI, calls []protocol.Range) (string, error) { - itemFile := conn.AddFile(ctx, item.URI.SpanURI()) + itemFile := conn.openFile(ctx, item.URI.SpanURI()) if itemFile.err != nil { return "", itemFile.err } @@ -123,7 +123,7 @@ func callItemPrintString(ctx context.Context, conn *connection, item protocol.Ca return "", err } - callsFile := conn.AddFile(ctx, callsURI.SpanURI()) + callsFile := conn.openFile(ctx, callsURI.SpanURI()) if callsURI != "" && callsFile.err != nil { return "", callsFile.err } diff --git a/gopls/internal/lsp/cmd/check.go b/gopls/internal/lsp/cmd/check.go index 5e4fe39eb36..cf081ca2615 100644 --- a/gopls/internal/lsp/cmd/check.go +++ b/gopls/internal/lsp/cmd/check.go @@ -48,7 +48,7 @@ func (c *check) Run(ctx context.Context, args ...string) error { for _, arg := range args { uri := span.URIFromPath(arg) uris = append(uris, uri) - file := conn.AddFile(ctx, uri) + file := conn.openFile(ctx, uri) if file.err != nil { return file.err } diff --git a/gopls/internal/lsp/cmd/cmd.go b/gopls/internal/lsp/cmd/cmd.go index 38accb6b9bb..5c64e108668 100644 --- a/gopls/internal/lsp/cmd/cmd.go +++ b/gopls/internal/lsp/cmd/cmd.go @@ -399,7 +399,7 @@ type cmdFile struct { uri span.URI mapper *protocol.ColumnMapper err error - added bool + open bool diagnostics []protocol.Diagnostic } @@ -558,22 +558,24 @@ func (c *cmdClient) getFile(ctx context.Context, uri span.URI) *cmdFile { return file } -func (c *connection) AddFile(ctx context.Context, uri span.URI) *cmdFile { - c.Client.filesMu.Lock() - defer c.Client.filesMu.Unlock() +func (c *cmdClient) openFile(ctx context.Context, uri span.URI) *cmdFile { + c.filesMu.Lock() + defer c.filesMu.Unlock() - file := c.Client.getFile(ctx, uri) - // This should never happen. - if file == nil { - return &cmdFile{ - uri: uri, - err: fmt.Errorf("no file found for %s", uri), - } + file := c.getFile(ctx, uri) + if file.err != nil || file.open { + return file } - if file.err != nil || file.added { + file.open = true + return file +} + +func (c *connection) openFile(ctx context.Context, uri span.URI) *cmdFile { + file := c.Client.openFile(ctx, uri) + if file.err != nil { return file } - file.added = true + p := &protocol.DidOpenTextDocumentParams{ TextDocument: protocol.TextDocumentItem{ URI: protocol.URIFromSpanURI(uri), diff --git a/gopls/internal/lsp/cmd/definition.go b/gopls/internal/lsp/cmd/definition.go index 9096e17153e..edfd7392902 100644 --- a/gopls/internal/lsp/cmd/definition.go +++ b/gopls/internal/lsp/cmd/definition.go @@ -80,7 +80,7 @@ func (d *definition) Run(ctx context.Context, args ...string) error { } defer conn.terminate(ctx) from := span.Parse(args[0]) - file := conn.AddFile(ctx, from.URI()) + file := conn.openFile(ctx, from.URI()) if file.err != nil { return file.err } @@ -113,7 +113,7 @@ func (d *definition) Run(ctx context.Context, args ...string) error { if hover == nil { return fmt.Errorf("%v: not an identifier", from) } - file = conn.AddFile(ctx, fileURI(locs[0].URI)) + file = conn.openFile(ctx, fileURI(locs[0].URI)) if file.err != nil { return fmt.Errorf("%v: %v", from, file.err) } diff --git a/gopls/internal/lsp/cmd/folding_range.go b/gopls/internal/lsp/cmd/folding_range.go index 17cead91f00..7a9cbf9e8fb 100644 --- a/gopls/internal/lsp/cmd/folding_range.go +++ b/gopls/internal/lsp/cmd/folding_range.go @@ -44,7 +44,7 @@ func (r *foldingRanges) Run(ctx context.Context, args ...string) error { defer conn.terminate(ctx) from := span.Parse(args[0]) - file := conn.AddFile(ctx, from.URI()) + file := conn.openFile(ctx, from.URI()) if file.err != nil { return file.err } diff --git a/gopls/internal/lsp/cmd/format.go b/gopls/internal/lsp/cmd/format.go index 1ad9614b476..2b8109c670a 100644 --- a/gopls/internal/lsp/cmd/format.go +++ b/gopls/internal/lsp/cmd/format.go @@ -57,7 +57,7 @@ func (c *format) Run(ctx context.Context, args ...string) error { defer conn.terminate(ctx) for _, arg := range args { spn := span.Parse(arg) - file := conn.AddFile(ctx, spn.URI()) + file := conn.openFile(ctx, spn.URI()) if file.err != nil { return file.err } diff --git a/gopls/internal/lsp/cmd/highlight.go b/gopls/internal/lsp/cmd/highlight.go index dcbd1097a06..0737e9c424a 100644 --- a/gopls/internal/lsp/cmd/highlight.go +++ b/gopls/internal/lsp/cmd/highlight.go @@ -47,7 +47,7 @@ func (r *highlight) Run(ctx context.Context, args ...string) error { defer conn.terminate(ctx) from := span.Parse(args[0]) - file := conn.AddFile(ctx, from.URI()) + file := conn.openFile(ctx, from.URI()) if file.err != nil { return file.err } diff --git a/gopls/internal/lsp/cmd/implementation.go b/gopls/internal/lsp/cmd/implementation.go index 259b72572b4..dbc5fc3223b 100644 --- a/gopls/internal/lsp/cmd/implementation.go +++ b/gopls/internal/lsp/cmd/implementation.go @@ -47,7 +47,7 @@ func (i *implementation) Run(ctx context.Context, args ...string) error { defer conn.terminate(ctx) from := span.Parse(args[0]) - file := conn.AddFile(ctx, from.URI()) + file := conn.openFile(ctx, from.URI()) if file.err != nil { return file.err } @@ -71,7 +71,7 @@ func (i *implementation) Run(ctx context.Context, args ...string) error { var spans []string for _, impl := range implementations { - f := conn.AddFile(ctx, fileURI(impl.URI)) + f := conn.openFile(ctx, fileURI(impl.URI)) span, err := f.mapper.Span(impl) if err != nil { return err diff --git a/gopls/internal/lsp/cmd/imports.go b/gopls/internal/lsp/cmd/imports.go index 5b741739421..fadc8466834 100644 --- a/gopls/internal/lsp/cmd/imports.go +++ b/gopls/internal/lsp/cmd/imports.go @@ -56,7 +56,7 @@ func (t *imports) Run(ctx context.Context, args ...string) error { from := span.Parse(args[0]) uri := from.URI() - file := conn.AddFile(ctx, uri) + file := conn.openFile(ctx, uri) if file.err != nil { return file.err } diff --git a/gopls/internal/lsp/cmd/links.go b/gopls/internal/lsp/cmd/links.go index aec36da910c..b5413bba59f 100644 --- a/gopls/internal/lsp/cmd/links.go +++ b/gopls/internal/lsp/cmd/links.go @@ -53,7 +53,7 @@ func (l *links) Run(ctx context.Context, args ...string) error { from := span.Parse(args[0]) uri := from.URI() - file := conn.AddFile(ctx, uri) + file := conn.openFile(ctx, uri) if file.err != nil { return file.err } diff --git a/gopls/internal/lsp/cmd/prepare_rename.go b/gopls/internal/lsp/cmd/prepare_rename.go index 434904b6920..e61bd622fe0 100644 --- a/gopls/internal/lsp/cmd/prepare_rename.go +++ b/gopls/internal/lsp/cmd/prepare_rename.go @@ -51,7 +51,7 @@ func (r *prepareRename) Run(ctx context.Context, args ...string) error { defer conn.terminate(ctx) from := span.Parse(args[0]) - file := conn.AddFile(ctx, from.URI()) + file := conn.openFile(ctx, from.URI()) if file.err != nil { return file.err } diff --git a/gopls/internal/lsp/cmd/references.go b/gopls/internal/lsp/cmd/references.go index bbebc9f917d..2abbb919299 100644 --- a/gopls/internal/lsp/cmd/references.go +++ b/gopls/internal/lsp/cmd/references.go @@ -51,7 +51,7 @@ func (r *references) Run(ctx context.Context, args ...string) error { defer conn.terminate(ctx) from := span.Parse(args[0]) - file := conn.AddFile(ctx, from.URI()) + file := conn.openFile(ctx, from.URI()) if file.err != nil { return file.err } @@ -74,7 +74,7 @@ func (r *references) Run(ctx context.Context, args ...string) error { } var spans []string for _, l := range locations { - f := conn.AddFile(ctx, fileURI(l.URI)) + f := conn.openFile(ctx, fileURI(l.URI)) // convert location to span for user-friendly 1-indexed line // and column numbers span, err := f.mapper.Span(l) diff --git a/gopls/internal/lsp/cmd/rename.go b/gopls/internal/lsp/cmd/rename.go index 48c67e3d30d..2cbd260febb 100644 --- a/gopls/internal/lsp/cmd/rename.go +++ b/gopls/internal/lsp/cmd/rename.go @@ -61,7 +61,7 @@ func (r *rename) Run(ctx context.Context, args ...string) error { defer conn.terminate(ctx) from := span.Parse(args[0]) - file := conn.AddFile(ctx, from.URI()) + file := conn.openFile(ctx, from.URI()) if file.err != nil { return file.err } @@ -92,7 +92,7 @@ func (r *rename) Run(ctx context.Context, args ...string) error { for _, u := range orderedURIs { uri := span.URIFromURI(u) - cmdFile := conn.AddFile(ctx, uri) + cmdFile := conn.openFile(ctx, uri) filename := cmdFile.uri.Filename() newContent, renameEdits, err := source.ApplyProtocolEdits(cmdFile.mapper, edits[uri]) diff --git a/gopls/internal/lsp/cmd/semantictokens.go b/gopls/internal/lsp/cmd/semantictokens.go index f90d49ccf34..3ed08d0248b 100644 --- a/gopls/internal/lsp/cmd/semantictokens.go +++ b/gopls/internal/lsp/cmd/semantictokens.go @@ -82,7 +82,7 @@ func (c *semtok) Run(ctx context.Context, args ...string) error { } defer conn.terminate(ctx) uri := span.URIFromPath(args[0]) - file := conn.AddFile(ctx, uri) + file := conn.openFile(ctx, uri) if file.err != nil { return file.err } diff --git a/gopls/internal/lsp/cmd/signature.go b/gopls/internal/lsp/cmd/signature.go index 657d77235f0..77805628ad0 100644 --- a/gopls/internal/lsp/cmd/signature.go +++ b/gopls/internal/lsp/cmd/signature.go @@ -46,7 +46,7 @@ func (r *signature) Run(ctx context.Context, args ...string) error { defer conn.terminate(ctx) from := span.Parse(args[0]) - file := conn.AddFile(ctx, from.URI()) + file := conn.openFile(ctx, from.URI()) if file.err != nil { return file.err } diff --git a/gopls/internal/lsp/cmd/suggested_fix.go b/gopls/internal/lsp/cmd/suggested_fix.go index 082e0481007..78310b3b3b9 100644 --- a/gopls/internal/lsp/cmd/suggested_fix.go +++ b/gopls/internal/lsp/cmd/suggested_fix.go @@ -56,7 +56,7 @@ func (s *suggestedFix) Run(ctx context.Context, args ...string) error { from := span.Parse(args[0]) uri := from.URI() - file := conn.AddFile(ctx, uri) + file := conn.openFile(ctx, uri) if file.err != nil { return file.err } diff --git a/gopls/internal/lsp/cmd/workspace_symbol.go b/gopls/internal/lsp/cmd/workspace_symbol.go index 71a121e3362..be1e24ef324 100644 --- a/gopls/internal/lsp/cmd/workspace_symbol.go +++ b/gopls/internal/lsp/cmd/workspace_symbol.go @@ -73,7 +73,7 @@ func (r *workspaceSymbol) Run(ctx context.Context, args ...string) error { return err } for _, s := range symbols { - f := conn.AddFile(ctx, fileURI(s.Location.URI)) + f := conn.openFile(ctx, fileURI(s.Location.URI)) span, err := f.mapper.Span(s.Location) if err != nil { return err diff --git a/gopls/internal/lsp/diagnostics.go b/gopls/internal/lsp/diagnostics.go index ff3e4b2ec24..a01898b26a4 100644 --- a/gopls/internal/lsp/diagnostics.go +++ b/gopls/internal/lsp/diagnostics.go @@ -97,6 +97,8 @@ func (d diagnosticSource) String() string { return "FromGoWork" case modCheckUpgradesSource: return "FromCheckForUpgrades" + case modVulncheckSource: + return "FromModVulncheck" default: return fmt.Sprintf("From?%d?", d) } @@ -217,6 +219,10 @@ func (s *Server) diagnose(ctx context.Context, snapshot source.Snapshot, forceAn defer done() // Wait for a free diagnostics slot. + // TODO(adonovan): opt: shouldn't it be the analysis implementation's + // job to de-dup and limit resource consumption? In any case this + // this function spends most its time waiting for awaitLoaded, at + // least initially. select { case <-ctx.Done(): return @@ -226,73 +232,62 @@ func (s *Server) diagnose(ctx context.Context, snapshot source.Snapshot, forceAn <-s.diagnosticsSema }() - // First, diagnose the go.mod file. - modReports, modErr := mod.Diagnostics(ctx, snapshot) - if ctx.Err() != nil { - log.Trace.Log(ctx, "diagnose cancelled") - return - } - if modErr != nil { - event.Error(ctx, "warning: diagnose go.mod", modErr, tag.Directory.Of(snapshot.View().Folder().Filename()), tag.Snapshot.Of(snapshot.ID())) - } - for id, diags := range modReports { - if id.URI == "" { - event.Error(ctx, "missing URI for module diagnostics", fmt.Errorf("empty URI"), tag.Directory.Of(snapshot.View().Folder().Filename())) - continue + // common code for dispatching diagnostics + store := func(dsource diagnosticSource, operation string, diagsByFileID map[source.VersionedFileIdentity][]*source.Diagnostic, err error) { + if err != nil { + event.Error(ctx, "warning: while "+operation, err, + tag.Directory.Of(snapshot.View().Folder().Filename()), + tag.Snapshot.Of(snapshot.ID())) + } + for id, diags := range diagsByFileID { + if id.URI == "" { + event.Error(ctx, "missing URI while "+operation, fmt.Errorf("empty URI"), tag.Directory.Of(snapshot.View().Folder().Filename())) + continue + } + s.storeDiagnostics(snapshot, id.URI, dsource, diags) } - s.storeDiagnostics(snapshot, id.URI, modSource, diags) } - upgradeModReports, upgradeErr := mod.UpgradeDiagnostics(ctx, snapshot) + + // Diagnose go.mod upgrades. + upgradeReports, upgradeErr := mod.UpgradeDiagnostics(ctx, snapshot) if ctx.Err() != nil { log.Trace.Log(ctx, "diagnose cancelled") return } - if upgradeErr != nil { - event.Error(ctx, "warning: diagnose go.mod upgrades", upgradeErr, tag.Directory.Of(snapshot.View().Folder().Filename()), tag.Snapshot.Of(snapshot.ID())) - } - for id, diags := range upgradeModReports { - if id.URI == "" { - event.Error(ctx, "missing URI for module diagnostics", fmt.Errorf("empty URI"), tag.Directory.Of(snapshot.View().Folder().Filename())) - continue - } - s.storeDiagnostics(snapshot, id.URI, modCheckUpgradesSource, diags) - } - vulnerabilityReports, vulnErr := mod.VulnerabilityDiagnostics(ctx, snapshot) + store(modCheckUpgradesSource, "diagnosing go.mod upgrades", upgradeReports, upgradeErr) + + // Diagnose vulnerabilities. + vulnReports, vulnErr := mod.VulnerabilityDiagnostics(ctx, snapshot) if ctx.Err() != nil { log.Trace.Log(ctx, "diagnose cancelled") return } - if vulnErr != nil { - event.Error(ctx, "warning: checking vulnerabilities", vulnErr, tag.Directory.Of(snapshot.View().Folder().Filename()), tag.Snapshot.Of(snapshot.ID())) - } - for id, diags := range vulnerabilityReports { - if id.URI == "" { - event.Error(ctx, "missing URI for module diagnostics", fmt.Errorf("empty URI"), tag.Directory.Of(snapshot.View().Folder().Filename())) - continue - } - s.storeDiagnostics(snapshot, id.URI, modVulncheckSource, diags) - } + store(modVulncheckSource, "diagnosing vulnerabilities", vulnReports, vulnErr) - // Diagnose the go.work file, if it exists. + // Diagnose go.work file. workReports, workErr := work.Diagnostics(ctx, snapshot) if ctx.Err() != nil { log.Trace.Log(ctx, "diagnose cancelled") return } - if workErr != nil { - event.Error(ctx, "warning: diagnose go.work", workErr, tag.Directory.Of(snapshot.View().Folder().Filename()), tag.Snapshot.Of(snapshot.ID())) - } - for id, diags := range workReports { - if id.URI == "" { - event.Error(ctx, "missing URI for work file diagnostics", fmt.Errorf("empty URI"), tag.Directory.Of(snapshot.View().Folder().Filename())) - continue - } - s.storeDiagnostics(snapshot, id.URI, workSource, diags) + store(workSource, "diagnosing go.work file", workReports, workErr) + + // All subsequent steps depend on the completion of + // type-checking of the all active packages in the workspace. + // This step may take many seconds initially. + // (mod.Diagnostics would implicitly wait for this too, + // but the control is clearer if it is explicit here.) + activePkgs, activeErr := snapshot.ActivePackages(ctx) + + // Diagnose go.mod file. + modReports, modErr := mod.Diagnostics(ctx, snapshot) + if ctx.Err() != nil { + log.Trace.Log(ctx, "diagnose cancelled") + return } + store(modSource, "diagnosing go.mod file", modReports, modErr) - // Diagnose all of the packages in the workspace. - wsPkgs, err := snapshot.ActivePackages(ctx) - if s.shouldIgnoreError(ctx, snapshot, err) { + if s.shouldIgnoreError(ctx, snapshot, activeErr) { return } criticalErr := snapshot.GetCriticalError(ctx) @@ -303,7 +298,7 @@ func (s *Server) diagnose(ctx context.Context, snapshot source.Snapshot, forceAn // error progress reports will be closed. s.showCriticalErrorStatus(ctx, snapshot, criticalErr) - // There may be .tmpl files. + // Diagnose template (.tmpl) files. for _, f := range snapshot.Templates() { diags := template.Diagnose(f) s.storeDiagnostics(snapshot, f.URI(), typeCheckSource, diags) @@ -311,29 +306,31 @@ func (s *Server) diagnose(ctx context.Context, snapshot source.Snapshot, forceAn // If there are no workspace packages, there is nothing to diagnose and // there are no orphaned files. - if len(wsPkgs) == 0 { + if len(activePkgs) == 0 { return } + // Run go/analysis diagnosis of packages in parallel. + // TODO(adonovan): opt: it may be more efficient to + // have diagnosePkg take a set of packages. var ( wg sync.WaitGroup seen = map[span.URI]struct{}{} ) - for _, pkg := range wsPkgs { - wg.Add(1) - + for _, pkg := range activePkgs { for _, pgf := range pkg.CompiledGoFiles() { seen[pgf.URI] = struct{}{} } + wg.Add(1) go func(pkg source.Package) { defer wg.Done() - s.diagnosePkg(ctx, snapshot, pkg, forceAnalysis) }(pkg) } wg.Wait() + // Orphaned files. // Confirm that every opened file belongs to a package (if any exist in // the workspace). Otherwise, add a diagnostic to the file. for _, o := range s.session.Overlays() { diff --git a/gopls/internal/lsp/fake/editor.go b/gopls/internal/lsp/fake/editor.go index dfd17c7e55a..f73301d674c 100644 --- a/gopls/internal/lsp/fake/editor.go +++ b/gopls/internal/lsp/fake/editor.go @@ -265,6 +265,10 @@ func (e *Editor) initialize(ctx context.Context) error { "event", "function", "method", "macro", "keyword", "modifier", "comment", "string", "number", "regexp", "operator", } + params.Capabilities.TextDocument.SemanticTokens.TokenModifiers = []string{ + "declaration", "definition", "readonly", "static", + "deprecated", "abstract", "async", "modification", "documentation", "defaultLibrary", + } // This is a bit of a hack, since the fake editor doesn't actually support // watching changed files that match a specific glob pattern. However, the diff --git a/gopls/internal/lsp/general.go b/gopls/internal/lsp/general.go index da4e4bfb89c..57348cd564b 100644 --- a/gopls/internal/lsp/general.go +++ b/gopls/internal/lsp/general.go @@ -226,11 +226,62 @@ func (s *Server) initialized(ctx context.Context, params *protocol.InitializedPa return nil } -// OldestSupportedGoVersion is the last X in Go 1.X that we support. +// GoVersionTable maps Go versions to the gopls version in which support will +// be deprecated, and the final gopls version supporting them without warnings. +// Keep this in sync with gopls/README.md // -// Mutable for testing, since we won't otherwise run CI on unsupported Go -// versions. -var OldestSupportedGoVersion = 16 +// Must be sorted in ascending order of Go version. +// +// Mutable for testing. +var GoVersionTable = []GoVersionSupport{ + {12, "", "v0.7.5"}, + {15, "v0.11.0", "v0.9.5"}, +} + +// GoVersionSupport holds information about end-of-life Go version support. +type GoVersionSupport struct { + GoVersion int + DeprecatedVersion string // if unset, the version is already deprecated + InstallGoplsVersion string +} + +// OldestSupportedGoVersion is the last X in Go 1.X that this version of gopls +// supports. +func OldestSupportedGoVersion() int { + return GoVersionTable[len(GoVersionTable)-1].GoVersion + 1 +} + +// versionMessage returns the warning/error message to display if the user is +// on the given Go version, if any. The goVersion variable is the X in Go 1.X. +// +// If goVersion is invalid (< 0), it returns "", 0. +func versionMessage(goVersion int) (string, protocol.MessageType) { + if goVersion < 0 { + return "", 0 + } + + for _, v := range GoVersionTable { + if goVersion <= v.GoVersion { + var msgBuilder strings.Builder + + mType := protocol.Error + fmt.Fprintf(&msgBuilder, "Found Go version 1.%d", goVersion) + if v.DeprecatedVersion != "" { + // not deprecated yet, just a warning + fmt.Fprintf(&msgBuilder, ", which will be unsupported by gopls %s. ", v.DeprecatedVersion) + mType = protocol.Warning + } else { + fmt.Fprint(&msgBuilder, ", which is not supported by this version of gopls. ") + } + fmt.Fprintf(&msgBuilder, "Please upgrade to Go 1.%d or later and reinstall gopls. ", OldestSupportedGoVersion()) + fmt.Fprintf(&msgBuilder, "If you can't upgrade and want this message to go away, please install gopls %s. ", v.InstallGoplsVersion) + fmt.Fprint(&msgBuilder, "See https://go.dev/s/gopls-support-policy for more details.") + + return msgBuilder.String(), mType + } + } + return "", 0 +} // checkViewGoVersions checks whether any Go version used by a view is too old, // raising a showMessage notification if so. @@ -245,10 +296,9 @@ func (s *Server) checkViewGoVersions() { } } - if oldestVersion >= 0 && oldestVersion < OldestSupportedGoVersion { - msg := fmt.Sprintf("Found Go version 1.%d, which is unsupported. Please upgrade to Go 1.%d or later.", oldestVersion, OldestSupportedGoVersion) + if msg, mType := versionMessage(oldestVersion); msg != "" { s.eventuallyShowMessage(context.Background(), &protocol.ShowMessageParams{ - Type: protocol.Error, + Type: mType, Message: msg, }) } @@ -258,18 +308,18 @@ func (s *Server) addFolders(ctx context.Context, folders []protocol.WorkspaceFol originalViews := len(s.session.Views()) viewErrors := make(map[span.URI]error) - var wg sync.WaitGroup + var ndiagnose sync.WaitGroup // number of unfinished diagnose calls if s.session.Options().VerboseWorkDoneProgress { work := s.progress.Start(ctx, DiagnosticWorkTitle(FromInitialWorkspaceLoad), "Calculating diagnostics for initial workspace load...", nil, nil) defer func() { go func() { - wg.Wait() + ndiagnose.Wait() work.End(ctx, "Done.") }() }() } // Only one view gets to have a workspace. - var allFoldersWg sync.WaitGroup + var nsnapshots sync.WaitGroup // number of unfinished snapshot initializations for _, folder := range folders { uri := span.URIFromURI(folder.URI) // Ignore non-file URIs. @@ -288,41 +338,40 @@ func (s *Server) addFolders(ctx context.Context, folders []protocol.WorkspaceFol } // Inv: release() must be called once. - var swg sync.WaitGroup - swg.Add(1) - allFoldersWg.Add(1) - // TODO(adonovan): this looks fishy. Is AwaitInitialized - // supposed to be called once per folder? - go func() { - defer swg.Done() - defer allFoldersWg.Done() - snapshot.AwaitInitialized(ctx) - work.End(ctx, "Finished loading packages.") - }() - // Print each view's environment. - buf := &bytes.Buffer{} - if err := snapshot.WriteEnv(ctx, buf); err != nil { + var buf bytes.Buffer + if err := snapshot.WriteEnv(ctx, &buf); err != nil { viewErrors[uri] = err release() continue } event.Log(ctx, buf.String()) - // Diagnose the newly created view. - wg.Add(1) + // Initialize snapshot asynchronously. + initialized := make(chan struct{}) + nsnapshots.Add(1) + go func() { + snapshot.AwaitInitialized(ctx) + work.End(ctx, "Finished loading packages.") + nsnapshots.Done() + close(initialized) // signal + }() + + // Diagnose the newly created view asynchronously. + ndiagnose.Add(1) go func() { s.diagnoseDetached(snapshot) - swg.Wait() + <-initialized release() - wg.Done() + ndiagnose.Done() }() } + // Wait for snapshots to be initialized so that all files are known. + // (We don't need to wait for diagnosis to finish.) + nsnapshots.Wait() + // Register for file watching notifications, if they are supported. - // Wait for all snapshots to be initialized first, since all files might - // not yet be known to the snapshots. - allFoldersWg.Wait() if err := s.updateWatchedDirectories(ctx); err != nil { event.Error(ctx, "failed to register for file watching notifications", err) } diff --git a/gopls/internal/lsp/general_test.go b/gopls/internal/lsp/general_test.go new file mode 100644 index 00000000000..a0312ba1b43 --- /dev/null +++ b/gopls/internal/lsp/general_test.go @@ -0,0 +1,44 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lsp + +import ( + "strings" + "testing" + + "golang.org/x/tools/gopls/internal/lsp/protocol" +) + +func TestVersionMessage(t *testing.T) { + tests := []struct { + goVersion int + wantContains []string // string fragments that we expect to see + wantType protocol.MessageType + }{ + {-1, nil, 0}, + {12, []string{"1.12", "not supported", "upgrade to Go 1.16", "install gopls v0.7.5"}, protocol.Error}, + {13, []string{"1.13", "will be unsupported by gopls v0.11.0", "upgrade to Go 1.16", "install gopls v0.9.5"}, protocol.Warning}, + {15, []string{"1.15", "will be unsupported by gopls v0.11.0", "upgrade to Go 1.16", "install gopls v0.9.5"}, protocol.Warning}, + {16, nil, 0}, + } + + for _, test := range tests { + gotMsg, gotType := versionMessage(test.goVersion) + + if len(test.wantContains) == 0 && gotMsg != "" { + t.Errorf("versionMessage(%d) = %q, want \"\"", test.goVersion, gotMsg) + } + + for _, want := range test.wantContains { + if !strings.Contains(gotMsg, want) { + t.Errorf("versionMessage(%d) = %q, want containing %q", test.goVersion, gotMsg, want) + } + } + + if gotType != test.wantType { + t.Errorf("versionMessage(%d) = returned message type %d, want %d", test.goVersion, gotType, test.wantType) + } + } +} diff --git a/gopls/internal/lsp/lsprpc/binder.go b/gopls/internal/lsp/lsprpc/binder.go index b12cc491ffb..01e59f7bb62 100644 --- a/gopls/internal/lsp/lsprpc/binder.go +++ b/gopls/internal/lsp/lsprpc/binder.go @@ -9,17 +9,17 @@ import ( "encoding/json" "fmt" + "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/internal/event" jsonrpc2_v2 "golang.org/x/tools/internal/jsonrpc2_v2" - "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/internal/xcontext" ) // The BinderFunc type adapts a bind function to implement the jsonrpc2.Binder // interface. -type BinderFunc func(ctx context.Context, conn *jsonrpc2_v2.Connection) (jsonrpc2_v2.ConnectionOptions, error) +type BinderFunc func(ctx context.Context, conn *jsonrpc2_v2.Connection) jsonrpc2_v2.ConnectionOptions -func (f BinderFunc) Bind(ctx context.Context, conn *jsonrpc2_v2.Connection) (jsonrpc2_v2.ConnectionOptions, error) { +func (f BinderFunc) Bind(ctx context.Context, conn *jsonrpc2_v2.Connection) jsonrpc2_v2.ConnectionOptions { return f(ctx, conn) } @@ -39,7 +39,7 @@ func NewServerBinder(newServer ServerFunc) *ServerBinder { return &ServerBinder{newServer: newServer} } -func (b *ServerBinder) Bind(ctx context.Context, conn *jsonrpc2_v2.Connection) (jsonrpc2_v2.ConnectionOptions, error) { +func (b *ServerBinder) Bind(ctx context.Context, conn *jsonrpc2_v2.Connection) jsonrpc2_v2.ConnectionOptions { client := protocol.ClientDispatcherV2(conn) server := b.newServer(ctx, client) serverHandler := protocol.ServerHandlerV2(server) @@ -55,7 +55,7 @@ func (b *ServerBinder) Bind(ctx context.Context, conn *jsonrpc2_v2.Connection) ( return jsonrpc2_v2.ConnectionOptions{ Handler: wrapped, Preempter: preempter, - }, nil + } } type canceler struct { @@ -94,13 +94,19 @@ func NewForwardBinder(dialer jsonrpc2_v2.Dialer) *ForwardBinder { } } -func (b *ForwardBinder) Bind(ctx context.Context, conn *jsonrpc2_v2.Connection) (opts jsonrpc2_v2.ConnectionOptions, _ error) { +func (b *ForwardBinder) Bind(ctx context.Context, conn *jsonrpc2_v2.Connection) (opts jsonrpc2_v2.ConnectionOptions) { client := protocol.ClientDispatcherV2(conn) clientBinder := NewClientBinder(func(context.Context, protocol.Server) protocol.Client { return client }) + serverConn, err := jsonrpc2_v2.Dial(context.Background(), b.dialer, clientBinder) if err != nil { - return opts, err + return jsonrpc2_v2.ConnectionOptions{ + Handler: jsonrpc2_v2.HandlerFunc(func(context.Context, *jsonrpc2_v2.Request) (interface{}, error) { + return nil, fmt.Errorf("%w: %v", jsonrpc2_v2.ErrInternal, err) + }), + } } + if b.onBind != nil { b.onBind(serverConn) } @@ -118,7 +124,7 @@ func (b *ForwardBinder) Bind(ctx context.Context, conn *jsonrpc2_v2.Connection) return jsonrpc2_v2.ConnectionOptions{ Handler: protocol.ServerHandlerV2(server), Preempter: preempter, - }, nil + } } // A ClientFunc is used to construct an LSP client for a given server. @@ -133,10 +139,10 @@ func NewClientBinder(newClient ClientFunc) *ClientBinder { return &ClientBinder{newClient} } -func (b *ClientBinder) Bind(ctx context.Context, conn *jsonrpc2_v2.Connection) (jsonrpc2_v2.ConnectionOptions, error) { +func (b *ClientBinder) Bind(ctx context.Context, conn *jsonrpc2_v2.Connection) jsonrpc2_v2.ConnectionOptions { server := protocol.ServerDispatcherV2(conn) client := b.newClient(ctx, server) return jsonrpc2_v2.ConnectionOptions{ Handler: protocol.ClientHandlerV2(client), - }, nil + } } diff --git a/gopls/internal/lsp/lsprpc/binder_test.go b/gopls/internal/lsp/lsprpc/binder_test.go index 8b048ab34e7..3315c3eb775 100644 --- a/gopls/internal/lsp/lsprpc/binder_test.go +++ b/gopls/internal/lsp/lsprpc/binder_test.go @@ -11,23 +11,20 @@ import ( "testing" "time" - jsonrpc2_v2 "golang.org/x/tools/internal/jsonrpc2_v2" "golang.org/x/tools/gopls/internal/lsp/protocol" + jsonrpc2_v2 "golang.org/x/tools/internal/jsonrpc2_v2" . "golang.org/x/tools/gopls/internal/lsp/lsprpc" ) type TestEnv struct { - Listeners []jsonrpc2_v2.Listener - Conns []*jsonrpc2_v2.Connection - Servers []*jsonrpc2_v2.Server + Conns []*jsonrpc2_v2.Connection + Servers []*jsonrpc2_v2.Server } func (e *TestEnv) Shutdown(t *testing.T) { - for _, l := range e.Listeners { - if err := l.Close(); err != nil { - t.Error(err) - } + for _, s := range e.Servers { + s.Shutdown() } for _, c := range e.Conns { if err := c.Close(); err != nil { @@ -46,11 +43,7 @@ func (e *TestEnv) serve(ctx context.Context, t *testing.T, server jsonrpc2_v2.Bi if err != nil { t.Fatal(err) } - e.Listeners = append(e.Listeners, l) - s, err := jsonrpc2_v2.Serve(ctx, l, server) - if err != nil { - t.Fatal(err) - } + s := jsonrpc2_v2.NewServer(ctx, l, server) e.Servers = append(e.Servers, s) return l, s } diff --git a/gopls/internal/lsp/lsprpc/commandinterceptor.go b/gopls/internal/lsp/lsprpc/commandinterceptor.go index be68efe78aa..607ee9c9e9f 100644 --- a/gopls/internal/lsp/lsprpc/commandinterceptor.go +++ b/gopls/internal/lsp/lsprpc/commandinterceptor.go @@ -8,8 +8,8 @@ import ( "context" "encoding/json" - jsonrpc2_v2 "golang.org/x/tools/internal/jsonrpc2_v2" "golang.org/x/tools/gopls/internal/lsp/protocol" + jsonrpc2_v2 "golang.org/x/tools/internal/jsonrpc2_v2" ) // HandlerMiddleware is a middleware that only modifies the jsonrpc2 handler. @@ -18,13 +18,10 @@ type HandlerMiddleware func(jsonrpc2_v2.Handler) jsonrpc2_v2.Handler // BindHandler transforms a HandlerMiddleware into a Middleware. func BindHandler(hmw HandlerMiddleware) Middleware { return Middleware(func(binder jsonrpc2_v2.Binder) jsonrpc2_v2.Binder { - return BinderFunc(func(ctx context.Context, conn *jsonrpc2_v2.Connection) (jsonrpc2_v2.ConnectionOptions, error) { - opts, err := binder.Bind(ctx, conn) - if err != nil { - return opts, err - } + return BinderFunc(func(ctx context.Context, conn *jsonrpc2_v2.Connection) jsonrpc2_v2.ConnectionOptions { + opts := binder.Bind(ctx, conn) opts.Handler = hmw(opts.Handler) - return opts, nil + return opts }) }) } diff --git a/gopls/internal/lsp/lsprpc/middleware.go b/gopls/internal/lsp/lsprpc/middleware.go index f703217dd0b..50089cde7dc 100644 --- a/gopls/internal/lsp/lsprpc/middleware.go +++ b/gopls/internal/lsp/lsprpc/middleware.go @@ -62,11 +62,8 @@ func (h *Handshaker) Peers() []PeerInfo { // Middleware is a jsonrpc2 middleware function to augment connection binding // to handle the handshake method, and record disconnections. func (h *Handshaker) Middleware(inner jsonrpc2_v2.Binder) jsonrpc2_v2.Binder { - return BinderFunc(func(ctx context.Context, conn *jsonrpc2_v2.Connection) (jsonrpc2_v2.ConnectionOptions, error) { - opts, err := inner.Bind(ctx, conn) - if err != nil { - return opts, err - } + return BinderFunc(func(ctx context.Context, conn *jsonrpc2_v2.Connection) jsonrpc2_v2.ConnectionOptions { + opts := inner.Bind(ctx, conn) localID := h.nextID() info := &PeerInfo{ @@ -93,7 +90,7 @@ func (h *Handshaker) Middleware(inner jsonrpc2_v2.Binder) jsonrpc2_v2.Binder { // Record the dropped client. go h.cleanupAtDisconnect(conn, localID) - return opts, nil + return opts }) } diff --git a/gopls/internal/lsp/lsprpc/middleware_test.go b/gopls/internal/lsp/lsprpc/middleware_test.go index a37294a31a1..c528eae5c62 100644 --- a/gopls/internal/lsp/lsprpc/middleware_test.go +++ b/gopls/internal/lsp/lsprpc/middleware_test.go @@ -11,12 +11,12 @@ import ( "testing" "time" - jsonrpc2_v2 "golang.org/x/tools/internal/jsonrpc2_v2" . "golang.org/x/tools/gopls/internal/lsp/lsprpc" + jsonrpc2_v2 "golang.org/x/tools/internal/jsonrpc2_v2" ) -var noopBinder = BinderFunc(func(context.Context, *jsonrpc2_v2.Connection) (jsonrpc2_v2.ConnectionOptions, error) { - return jsonrpc2_v2.ConnectionOptions{}, nil +var noopBinder = BinderFunc(func(context.Context, *jsonrpc2_v2.Connection) jsonrpc2_v2.ConnectionOptions { + return jsonrpc2_v2.ConnectionOptions{} }) func TestHandshakeMiddleware(t *testing.T) { diff --git a/gopls/internal/lsp/mod/diagnostics.go b/gopls/internal/lsp/mod/diagnostics.go index 4b4f421ece1..770fb3729f7 100644 --- a/gopls/internal/lsp/mod/diagnostics.go +++ b/gopls/internal/lsp/mod/diagnostics.go @@ -7,7 +7,6 @@ package mod import ( - "bytes" "context" "fmt" "sort" @@ -25,6 +24,8 @@ import ( ) // Diagnostics returns diagnostics for the modules in the workspace. +// +// It waits for completion of type-checking of all active packages. func Diagnostics(ctx context.Context, snapshot source.Snapshot) (map[source.VersionedFileIdentity][]*source.Diagnostic, error) { ctx, done := event.Start(ctx, "mod.Diagnostics", tag.Snapshot.Of(snapshot.ID())) defer done() @@ -73,8 +74,9 @@ func collectDiagnostics(ctx context.Context, snapshot source.Snapshot, diagFn fu return reports, nil } -// ModDiagnostics returns diagnostics from diagnosing the packages in the workspace and -// from tidying the go.mod file. +// ModDiagnostics waits for completion of type-checking of all active +// packages, then returns diagnostics from diagnosing the packages in +// the workspace and from tidying the go.mod file. func ModDiagnostics(ctx context.Context, snapshot source.Snapshot, fh source.FileHandle) (diagnostics []*source.Diagnostic, err error) { pm, err := snapshot.ParseMod(ctx, fh) if err != nil { @@ -195,29 +197,49 @@ func ModVulnerabilityDiagnostics(ctx context.Context, snapshot source.Snapshot, vs := snapshot.View().Vulnerabilities(fh.URI()) // TODO(suzmue): should we just store the vulnerabilities like this? - vulns := make(map[string][]govulncheck.Vuln) + affecting := make(map[string][]govulncheck.Vuln) + nonaffecting := make(map[string][]govulncheck.Vuln) for _, v := range vs { - vulns[v.ModPath] = append(vulns[v.ModPath], v) + if len(v.Trace) > 0 { + affecting[v.ModPath] = append(affecting[v.ModPath], v) + } else { + nonaffecting[v.ModPath] = append(nonaffecting[v.ModPath], v) + } } for _, req := range pm.File.Require { - vulnList, ok := vulns[req.Mod.Path] - if !ok { + affectingVulns, ok := affecting[req.Mod.Path] + nonaffectingVulns, ok2 := nonaffecting[req.Mod.Path] + if !ok && !ok2 { continue } rng, err := pm.Mapper.OffsetRange(req.Syntax.Start.Byte, req.Syntax.End.Byte) if err != nil { return nil, err } - for _, v := range vulnList { + // Map affecting vulns to 'warning' level diagnostics, + // others to 'info' level diagnostics. + // Fixes will include only the upgrades for warning level diagnostics. + var fixes []source.SuggestedFix + var warning, info []string + for _, v := range nonaffectingVulns { + // Only show the diagnostic if the vulnerability was calculated + // for the module at the current version. + if semver.IsValid(v.FoundIn) && semver.Compare(req.Mod.Version, v.FoundIn) != 0 { + continue + } + info = append(info, v.OSV.ID) + } + for _, v := range affectingVulns { // Only show the diagnostic if the vulnerability was calculated // for the module at the current version. if semver.IsValid(v.FoundIn) && semver.Compare(req.Mod.Version, v.FoundIn) != 0 { continue } + warning = append(warning, v.OSV.ID) // Upgrade to the exact version we offer the user, not the most recent. // TODO(hakim): Produce fixes only for affecting vulnerabilities (if len(v.Trace) > 0) - var fixes []source.SuggestedFix + if fixedVersion := v.FixedIn; semver.IsValid(fixedVersion) && semver.Compare(req.Mod.Version, fixedVersion) < 0 { cmd, err := getUpgradeCodeAction(fh, req, fixedVersion) if err != nil { @@ -235,24 +257,41 @@ func ModVulnerabilityDiagnostics(ctx context.Context, snapshot source.Snapshot, source.SuggestedFixFromCommand(latest, protocol.QuickFix), } } + } - severity := protocol.SeverityInformation - if len(v.Trace) > 0 { - severity = protocol.SeverityWarning - } + if len(warning) == 0 && len(info) == 0 { + return nil, nil + } + severity := protocol.SeverityInformation + if len(warning) > 0 { + severity = protocol.SeverityWarning + } - vulnDiagnostics = append(vulnDiagnostics, &source.Diagnostic{ - URI: fh.URI(), - Range: rng, - Severity: severity, - Source: source.Vulncheck, - Code: v.OSV.ID, - CodeHref: href(v.OSV), - Message: formatMessage(v), - SuggestedFixes: fixes, - }) + sort.Strings(warning) + sort.Strings(info) + + var b strings.Builder + if len(warning) == 1 { + fmt.Fprintf(&b, "%v has a vulnerability used in the code: %v.", req.Mod.Path, warning[0]) + } else { + fmt.Fprintf(&b, "%v has vulnerabilities used in the code: %v.", req.Mod.Path, strings.Join(warning, ", ")) + } + if len(warning) == 0 { + if len(info) == 1 { + fmt.Fprintf(&b, "%v has a vulnerability %v that is not used in the code.", req.Mod.Path, info[0]) + } else { + fmt.Fprintf(&b, "%v has known vulnerabilities %v that are not used in the code.", req.Mod.Path, strings.Join(info, ", ")) + } } + vulnDiagnostics = append(vulnDiagnostics, &source.Diagnostic{ + URI: fh.URI(), + Range: rng, + Severity: severity, + Source: source.Vulncheck, + Message: b.String(), + SuggestedFixes: fixes, + }) } return vulnDiagnostics, nil @@ -266,7 +305,7 @@ func formatMessage(v govulncheck.Vuln) string { details[i] = ' ' } } - return fmt.Sprintf("%s has a known vulnerability: %s", v.ModPath, string(bytes.TrimSpace(details))) + return strings.TrimSpace(strings.Replace(string(details), "\n\n", "\n\n ", -1)) } // href returns a URL embedded in the entry if any. diff --git a/gopls/internal/lsp/mod/hover.go b/gopls/internal/lsp/mod/hover.go index 29812749df3..5d5b6158212 100644 --- a/gopls/internal/lsp/mod/hover.go +++ b/gopls/internal/lsp/mod/hover.go @@ -8,9 +8,11 @@ import ( "bytes" "context" "fmt" + "sort" "strings" "golang.org/x/mod/modfile" + "golang.org/x/tools/gopls/internal/govulncheck" "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/internal/event" @@ -55,18 +57,22 @@ func Hover(ctx context.Context, snapshot source.Snapshot, fh source.FileHandle, } // Shift the start position to the location of the // dependency within the require statement. - startPos, endPos = s+i, s+i+len(dep) + startPos, endPos = s+i, e if startPos <= offset && offset <= endPos { req = r break } } + // TODO(hyangah): find position for info about vulnerabilities in Go // The cursor position is not on a require statement. if req == nil { return nil, nil } + // Get the vulnerability info. + affecting, nonaffecting := lookupVulns(snapshot.View().Vulnerabilities(fh.URI()), req) + // Get the `go mod why` results for the given file. why, err := snapshot.ModWhy(ctx, fh) if err != nil { @@ -78,38 +84,120 @@ func Hover(ctx context.Context, snapshot source.Snapshot, fh source.FileHandle, } // Get the range to highlight for the hover. + // TODO(hyangah): adjust the hover range to include the version number + // to match the diagnostics' range. rng, err := pm.Mapper.OffsetRange(startPos, endPos) if err != nil { return nil, err } - if err != nil { - return nil, err - } options := snapshot.View().Options() isPrivate := snapshot.View().IsGoPrivatePath(req.Mod.Path) + header := formatHeader(req.Mod.Path, options) explanation = formatExplanation(explanation, req, options, isPrivate) + vulns := formatVulnerabilities(affecting, nonaffecting, options) + return &protocol.Hover{ Contents: protocol.MarkupContent{ Kind: options.PreferredContentFormat, - Value: explanation, + Value: header + vulns + explanation, }, Range: rng, }, nil } -func formatExplanation(text string, req *modfile.Require, options *source.Options, isPrivate bool) string { - text = strings.TrimSuffix(text, "\n") - splt := strings.Split(text, "\n") - length := len(splt) - +func formatHeader(modpath string, options *source.Options) string { var b strings.Builder // Write the heading as an H3. - b.WriteString("##" + splt[0]) + b.WriteString("#### " + modpath) if options.PreferredContentFormat == protocol.Markdown { b.WriteString("\n\n") } else { b.WriteRune('\n') } + return b.String() +} + +func compareVuln(i, j govulncheck.Vuln) bool { + if i.OSV.ID == j.OSV.ID { + return i.PkgPath < j.PkgPath + } + return i.OSV.ID < j.OSV.ID +} + +func lookupVulns(vulns []govulncheck.Vuln, req *modfile.Require) (affecting, nonaffecting []govulncheck.Vuln) { + modpath, modversion := req.Mod.Path, req.Mod.Version + + var info, warning []govulncheck.Vuln + for _, vuln := range vulns { + if vuln.ModPath != modpath || vuln.FoundIn != modversion { + continue + } + if len(vuln.Trace) == 0 { + info = append(info, vuln) + } else { + warning = append(warning, vuln) + } + } + sort.Slice(info, func(i, j int) bool { return compareVuln(info[i], info[j]) }) + sort.Slice(warning, func(i, j int) bool { return compareVuln(warning[i], warning[j]) }) + return warning, info +} + +func formatVulnerabilities(affecting, nonaffecting []govulncheck.Vuln, options *source.Options) string { + if len(affecting) == 0 && len(nonaffecting) == 0 { + return "" + } + + // TODO(hyangah): can we use go templates to generate hover messages? + // Then, we can use a different template for markdown case. + useMarkdown := options.PreferredContentFormat == protocol.Markdown + + var b strings.Builder + + if len(affecting) > 0 { + // TODO(hyangah): make the message more eyecatching (icon/codicon/color) + if len(affecting) == 1 { + b.WriteString(fmt.Sprintf("\n**WARNING:** Found %d reachable vulnerability.\n", len(affecting))) + } else { + b.WriteString(fmt.Sprintf("\n**WARNING:** Found %d reachable vulnerabilities.\n", len(affecting))) + } + } + for _, v := range affecting { + fix := "No fix is available." + if v.FixedIn != "" { + fix = "Fixed in " + v.FixedIn + "." + } + + if useMarkdown { + fmt.Fprintf(&b, "- [**%v**](%v) %v %v\n", v.OSV.ID, href(v.OSV), formatMessage(v), fix) + } else { + fmt.Fprintf(&b, " - [%v] %v (%v) %v\n", v.OSV.ID, formatMessage(v), href(v.OSV), fix) + } + } + if len(nonaffecting) > 0 { + fmt.Fprintf(&b, "\n**FYI:** The project imports packages with known vulnerabilities, but does not call the vulnerable code.\n") + } + for _, v := range nonaffecting { + fix := "No fix is available." + if v.FixedIn != "" { + fix = "Fixed in " + v.FixedIn + "." + } + if useMarkdown { + fmt.Fprintf(&b, "- [%v](%v) %v %v\n", v.OSV.ID, href(v.OSV), formatMessage(v), fix) + } else { + fmt.Fprintf(&b, " - [%v] %v %v (%v)\n", v.OSV.ID, formatMessage(v), fix, href(v.OSV)) + } + } + b.WriteString("\n") + return b.String() +} + +func formatExplanation(text string, req *modfile.Require, options *source.Options, isPrivate bool) string { + text = strings.TrimSuffix(text, "\n") + splt := strings.Split(text, "\n") + length := len(splt) + + var b strings.Builder // If the explanation is 2 lines, then it is of the form: // # golang.org/x/text/encoding diff --git a/gopls/internal/lsp/protocol/generate/compare.go b/gopls/internal/lsp/protocol/generate/compare.go new file mode 100644 index 00000000000..d341307821d --- /dev/null +++ b/gopls/internal/lsp/protocol/generate/compare.go @@ -0,0 +1,10 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.19 +// +build go1.19 + +package main + +// compare the generated files in two directories diff --git a/gopls/internal/lsp/protocol/generate/data.go b/gopls/internal/lsp/protocol/generate/data.go new file mode 100644 index 00000000000..435f594bcf7 --- /dev/null +++ b/gopls/internal/lsp/protocol/generate/data.go @@ -0,0 +1,104 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.19 +// +build go1.19 + +package main + +// various data tables + +// methodNames is a map from the method to the name of the function that handles it +var methodNames = map[string]string{ + "$/cancelRequest": "CancelRequest", + "$/logTrace": "LogTrace", + "$/progress": "Progress", + "$/setTrace": "SetTrace", + "callHierarchy/incomingCalls": "IncomingCalls", + "callHierarchy/outgoingCalls": "OutgoingCalls", + "client/registerCapability": "RegisterCapability", + "client/unregisterCapability": "UnregisterCapability", + "codeAction/resolve": "ResolveCodeAction", + "codeLens/resolve": "ResolveCodeLens", + "completionItem/resolve": "ResolveCompletionItem", + "documentLink/resolve": "ResolveDocumentLink", + "exit": "Exit", + "initialize": "Initialize", + "initialized": "Initialized", + "inlayHint/resolve": "Resolve", + "notebookDocument/didChange": "DidChangeNotebookDocument", + "notebookDocument/didClose": "DidCloseNotebookDocument", + "notebookDocument/didOpen": "DidOpenNotebookDocument", + "notebookDocument/didSave": "DidSaveNotebookDocument", + "shutdown": "Shutdown", + "telemetry/event": "Event", + "textDocument/codeAction": "CodeAction", + "textDocument/codeLens": "CodeLens", + "textDocument/colorPresentation": "ColorPresentation", + "textDocument/completion": "Completion", + "textDocument/declaration": "Declaration", + "textDocument/definition": "Definition", + "textDocument/diagnostic": "Diagnostic", + "textDocument/didChange": "DidChange", + "textDocument/didClose": "DidClose", + "textDocument/didOpen": "DidOpen", + "textDocument/didSave": "DidSave", + "textDocument/documentColor": "DocumentColor", + "textDocument/documentHighlight": "DocumentHighlight", + "textDocument/documentLink": "DocumentLink", + "textDocument/documentSymbol": "DocumentSymbol", + "textDocument/foldingRange": "FoldingRange", + "textDocument/formatting": "Formatting", + "textDocument/hover": "Hover", + "textDocument/implementation": "Implementation", + "textDocument/inlayHint": "InlayHint", + "textDocument/inlineValue": "InlineValue", + "textDocument/linkedEditingRange": "LinkedEditingRange", + "textDocument/moniker": "Moniker", + "textDocument/onTypeFormatting": "OnTypeFormatting", + "textDocument/prepareCallHierarchy": "PrepareCallHierarchy", + "textDocument/prepareRename": "PrepareRename", + "textDocument/prepareTypeHierarchy": "PrepareTypeHierarchy", + "textDocument/publishDiagnostics": "PublishDiagnostics", + "textDocument/rangeFormatting": "RangeFormatting", + "textDocument/references": "References", + "textDocument/rename": "Rename", + "textDocument/selectionRange": "SelectionRange", + "textDocument/semanticTokens/full": "SemanticTokensFull", + "textDocument/semanticTokens/full/delta": "SemanticTokensFullDelta", + "textDocument/semanticTokens/range": "SemanticTokensRange", + "textDocument/signatureHelp": "SignatureHelp", + "textDocument/typeDefinition": "TypeDefinition", + "textDocument/willSave": "WillSave", + "textDocument/willSaveWaitUntil": "WillSaveWaitUntil", + "typeHierarchy/subtypes": "Subtypes", + "typeHierarchy/supertypes": "Supertypes", + "window/logMessage": "LogMessage", + "window/showDocument": "ShowDocument", + "window/showMessage": "ShowMessage", + "window/showMessageRequest": "ShowMessageRequest", + "window/workDoneProgress/cancel": "WorkDoneProgressCancel", + "window/workDoneProgress/create": "WorkDoneProgressCreate", + "workspace/applyEdit": "ApplyEdit", + "workspace/codeLens/refresh": "CodeLensRefresh", + "workspace/configuration": "Configuration", + "workspace/diagnostic": "DiagnosticWorkspace", + "workspace/diagnostic/refresh": "DiagnosticRefresh", + "workspace/didChangeConfiguration": "DidChangeConfiguration", + "workspace/didChangeWatchedFiles": "DidChangeWatchedFiles", + "workspace/didChangeWorkspaceFolders": "DidChangeWorkspaceFolders", + "workspace/didCreateFiles": "DidCreateFiles", + "workspace/didDeleteFiles": "DidDeleteFiles", + "workspace/didRenameFiles": "DidRenameFiles", + "workspace/executeCommand": "ExecuteCommand", + "workspace/inlayHint/refresh": "InlayHintRefresh", + "workspace/inlineValue/refresh": "InlineValueRefresh", + "workspace/semanticTokens/refresh": "SemanticTokensRefresh", + "workspace/symbol": "Symbol", + "workspace/willCreateFiles": "WillCreateFiles", + "workspace/willDeleteFiles": "WillDeleteFiles", + "workspace/willRenameFiles": "WillRenameFiles", + "workspace/workspaceFolders": "WorkspaceFolders", + "workspaceSymbol/resolve": "ResolveWorkspaceSymbol", +} diff --git a/gopls/internal/lsp/protocol/generate/doc.go b/gopls/internal/lsp/protocol/generate/doc.go new file mode 100644 index 00000000000..74685559c8e --- /dev/null +++ b/gopls/internal/lsp/protocol/generate/doc.go @@ -0,0 +1,32 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.19 +// +build go1.19 + +/* +GenLSP generates the files tsprotocol.go, tsclient.go, +tsserver.go, tsjson.go that support the language server protocol +for gopls. + +Usage: + + go run . [flags] + +The flags are: + + -d + The directory containing the vscode-languageserver-node repository. + (git clone https://github.com/microsoft/vscode-languageserver-node.git). + If not specified, the default is $HOME/vscode-languageserver-node. + + -o + The directory to write the generated files to. It must exist. + The default is "gen". + + -c + Compare the generated files to the files in the specified directory. + If this flag is not specified, no comparison is done. +*/ +package main diff --git a/gopls/internal/lsp/protocol/generate/generate.go b/gopls/internal/lsp/protocol/generate/generate.go new file mode 100644 index 00000000000..86c332856af --- /dev/null +++ b/gopls/internal/lsp/protocol/generate/generate.go @@ -0,0 +1,10 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.19 +// +build go1.19 + +package main + +// generate the Go code diff --git a/gopls/internal/lsp/protocol/generate/main.go b/gopls/internal/lsp/protocol/generate/main.go new file mode 100644 index 00000000000..38d25705d32 --- /dev/null +++ b/gopls/internal/lsp/protocol/generate/main.go @@ -0,0 +1,93 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.19 +// +build go1.19 + +package main + +import ( + "flag" + "fmt" + "log" + "os" +) + +var ( + // git clone https://github.com/microsoft/vscode-languageserver-node.git + repodir = flag.String("d", "", "directory of vscode-languageserver-node") + outputdir = flag.String("o", "gen", "output directory") + cmpolder = flag.String("c", "", "directory of older generated code") +) + +func main() { + log.SetFlags(log.Lshortfile) // log file name and line number, not time + flag.Parse() + + if *repodir == "" { + *repodir = fmt.Sprintf("%s/vscode-languageserver-node", os.Getenv("HOME")) + } + spec := parse(*repodir) + + // index the information in the specification + spec.indexRPCInfo() // messages + spec.indexDefInfo() // named types + +} + +func (s *spec) indexRPCInfo() { + for _, r := range s.model.Requests { + r := r + s.byMethod[r.Method] = &r + } + for _, n := range s.model.Notifications { + n := n + if n.Method == "$/cancelRequest" { + // viewed as too confusing to generate + continue + } + s.byMethod[n.Method] = &n + } +} + +func (sp *spec) indexDefInfo() { + for _, s := range sp.model.Structures { + s := s + sp.byName[s.Name] = &s + } + for _, e := range sp.model.Enumerations { + e := e + sp.byName[e.Name] = &e + } + for _, ta := range sp.model.TypeAliases { + ta := ta + sp.byName[ta.Name] = &ta + } + + // some Structure and TypeAlias names need to be changed for Go + // so byName contains the name used in the .json file, and + // the Name field contains the Go version of the name. + v := sp.model.Structures + for i, s := range v { + switch s.Name { + case "_InitializeParams": // _ is not upper case + v[i].Name = "XInitializeParams" + case "ConfigurationParams": // gopls compatibility + v[i].Name = "ParamConfiguration" + case "InitializeParams": // gopls compatibility + v[i].Name = "ParamInitialize" + case "PreviousResultId": // Go naming convention + v[i].Name = "PreviousResultID" + case "WorkspaceFoldersServerCapabilities": // gopls compatibility + v[i].Name = "WorkspaceFolders5Gn" + } + } + w := sp.model.TypeAliases + for i, t := range w { + switch t.Name { + case "PrepareRenameResult": // gopls compatibility + w[i].Name = "PrepareRename2Gn" + } + } +} diff --git a/gopls/internal/lsp/protocol/generate/main_test.go b/gopls/internal/lsp/protocol/generate/main_test.go new file mode 100644 index 00000000000..d986b59cee9 --- /dev/null +++ b/gopls/internal/lsp/protocol/generate/main_test.go @@ -0,0 +1,122 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.19 +// +build go1.19 + +package main + +import ( + "encoding/json" + "fmt" + "log" + "os" + "testing" +) + +// this is not a test, but an easy way to invoke the debugger +func TestAll(t *testing.T) { + t.Skip("run by hand") + log.SetFlags(log.Lshortfile) + main() +} + +// this is not a test, but an easy way to invoke the debugger +func TestCompare(t *testing.T) { + t.Skip("run by hand") + log.SetFlags(log.Lshortfile) + *cmpolder = "../lsp/gen" // instead use a directory containing the older generated files + main() +} + +// check that the parsed file includes all the information +// from the json file. This test will fail if the spec +// introduces new fields. (one can test this test by +// commenting out some special handling in parse.go.) +func TestParseContents(t *testing.T) { + t.Skip("run by hand") + log.SetFlags(log.Lshortfile) + + // compute our parse of the specification + dir := os.Getenv("HOME") + "/vscode-languageserver-node" + v := parse(dir) + out, err := json.Marshal(v.model) + if err != nil { + t.Fatal(err) + } + var our interface{} + if err := json.Unmarshal(out, &our); err != nil { + t.Fatal(err) + } + + // process the json file + fname := dir + "/protocol/metaModel.json" + buf, err := os.ReadFile(fname) + if err != nil { + t.Fatalf("could not read metaModel.json: %v", err) + } + var raw interface{} + if err := json.Unmarshal(buf, &raw); err != nil { + t.Fatal(err) + } + + // convert to strings showing the fields + them := flatten(raw) + us := flatten(our) + + // everything in them should be in us + lesser := make(sortedMap[bool]) + for _, s := range them { + lesser[s] = true + } + greater := make(sortedMap[bool]) // set of fields we have + for _, s := range us { + greater[s] = true + } + for _, k := range lesser.keys() { // set if fields they have + if !greater[k] { + t.Errorf("missing %s", k) + } + } +} + +// flatten(nil) = "nil" +// flatten(v string) = fmt.Sprintf("%q", v) +// flatten(v float64)= fmt.Sprintf("%g", v) +// flatten(v bool) = fmt.Sprintf("%v", v) +// flatten(v []any) = []string{"[0]"flatten(v[0]), "[1]"flatten(v[1]), ...} +// flatten(v map[string]any) = {"key1": flatten(v["key1"]), "key2": flatten(v["key2"]), ...} +func flatten(x any) []string { + switch v := x.(type) { + case nil: + return []string{"nil"} + case string: + return []string{fmt.Sprintf("%q", v)} + case float64: + return []string{fmt.Sprintf("%g", v)} + case bool: + return []string{fmt.Sprintf("%v", v)} + case []any: + var ans []string + for i, x := range v { + idx := fmt.Sprintf("[%.3d]", i) + for _, s := range flatten(x) { + ans = append(ans, idx+s) + } + } + return ans + case map[string]any: + var ans []string + for k, x := range v { + idx := fmt.Sprintf("%q:", k) + for _, s := range flatten(x) { + ans = append(ans, idx+s) + } + } + return ans + default: + log.Fatalf("unexpected type %T", x) + return nil + } +} diff --git a/gopls/internal/lsp/protocol/generate/naming.go b/gopls/internal/lsp/protocol/generate/naming.go new file mode 100644 index 00000000000..9d9201a49d9 --- /dev/null +++ b/gopls/internal/lsp/protocol/generate/naming.go @@ -0,0 +1,64 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.19 +// +build go1.19 + +package main + +// assign names to types. many types come with names, but names +// have to be provided for "or", "and", "tuple", and "literal" types. +// Only one tuple type occurs, so it poses no problem. Otherwise +// the name cannot depend on the ordering of the components, as permuting +// them doesn't change the type. One possibility is to build the name +// of the type out of the names of its components, done in an +// earlier version of this code, but rejected by code reviewers. +// (the name would change if the components changed.) +// An alternate is to use the definition context, which is what is done here +// and works for the existing code. However, it cannot work in general. +// (This easiest case is an "or" type with two "literal" components. +// The components will get the same name, as their definition contexts +// are identical.) spec.byName contains enough information to detect +// such cases. (Note that sometimes giving the same name to different +// types is correct, for instance when they involve stringLiterals.) + +import ( + "strings" +) + +// stacks contain information about the ancestry of a type +// (spaces and initial capital letters are treated specially in stack.name()) +type stack []string + +func (s stack) push(v string) stack { + return append(s, v) +} + +func (s stack) pop() { + s = s[:len(s)-1] +} + +// generate a type name from the stack that contains its ancestry +// +// For instance, ["Result textDocument/implementation"] becomes "_textDocument_implementation" +// which, after being returned, becomes "Or_textDocument_implementation", +// which will become "[]Location" eventually (for gopls compatibility). +func (s stack) name(prefix string) string { + var nm string + var seen int + // use the most recent 2 entries, if there are 2, + // or just the only one. + for i := len(s) - 1; i >= 0 && seen < 2; i-- { + x := s[i] + if x[0] <= 'Z' && x[0] >= 'A' { + // it may contain a message + if idx := strings.Index(x, " "); idx >= 0 { + x = prefix + strings.Replace(x[idx+1:], "/", "_", -1) + } + nm += x + seen++ + } + } + return nm +} diff --git a/gopls/internal/lsp/protocol/generate/output.go b/gopls/internal/lsp/protocol/generate/output.go new file mode 100644 index 00000000000..14a04864e85 --- /dev/null +++ b/gopls/internal/lsp/protocol/generate/output.go @@ -0,0 +1,10 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.19 +// +build go1.19 + +package main + +// Write the output diff --git a/gopls/internal/lsp/protocol/generate/parse.go b/gopls/internal/lsp/protocol/generate/parse.go new file mode 100644 index 00000000000..9f8067eff4d --- /dev/null +++ b/gopls/internal/lsp/protocol/generate/parse.go @@ -0,0 +1,174 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.19 +// +build go1.19 + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "time" +) + +// a spec contains the specification of the protocol, and derived information. +type spec struct { + model *Model + + // combined Requests and Notifications, indexed by method (e.g., "textDocument/didOpen") + byMethod sortedMap[Message] + + // Structures, Enumerations, and TypeAliases, indexed by name used in + // the .json specification file + // (Some Structure and Enumeration names need to be changed for Go, + // such as _Initialize) + byName sortedMap[Defined] + + // computed type information + nameToTypes sortedMap[[]*Type] // all the uses of a type name + + // remember which types are in a union type + orTypes sortedMap[sortedMap[bool]] + + // information about the version of vscode-languageclient-node + githash string + modTime time.Time +} + +// parse the specification file and return a spec. +// (TestParseContents checks that the parse gets all the fields of the specification) +func parse(dir string) *spec { + fname := filepath.Join(dir, "protocol", "metaModel.json") + buf, err := os.ReadFile(fname) + if err != nil { + log.Fatalf("could not read metaModel.json: %v", err) + } + // line numbers in the .json file occur as comments in tsprotocol.go + newbuf := addLineNumbers(buf) + var v Model + if err := json.Unmarshal(newbuf, &v); err != nil { + log.Fatalf("could not unmarshal metaModel.json: %v", err) + } + + ans := &spec{ + model: &v, + byMethod: make(sortedMap[Message]), + byName: make(sortedMap[Defined]), + nameToTypes: make(sortedMap[[]*Type]), + orTypes: make(sortedMap[sortedMap[bool]]), + } + ans.githash, ans.modTime = gitInfo(dir) + return ans +} + +// gitInfo returns the git hash and modtime of the repository. +func gitInfo(dir string) (string, time.Time) { + fname := dir + "/.git/HEAD" + buf, err := os.ReadFile(fname) + if err != nil { + log.Fatal(err) + } + buf = bytes.TrimSpace(buf) + var githash string + if len(buf) == 40 { + githash = string(buf[:40]) + } else if bytes.HasPrefix(buf, []byte("ref: ")) { + fname = dir + "/.git/" + string(buf[5:]) + buf, err = os.ReadFile(fname) + if err != nil { + log.Fatal(err) + } + githash = string(buf[:40]) + } else { + log.Fatalf("githash cannot be recovered from %s", fname) + } + loadTime := time.Now() + return githash, loadTime +} + +// addLineNumbers adds a "line" field to each object in the JSON. +func addLineNumbers(buf []byte) []byte { + var ans []byte + // In the specification .json file, the delimiter '{' is + // always followed by a newline. There are other {s embedded in strings. + // json.Token does not return \n, or :, or , so using it would + // require parsing the json to reconstruct the missing information. + for linecnt, i := 1, 0; i < len(buf); i++ { + ans = append(ans, buf[i]) + switch buf[i] { + case '{': + if buf[i+1] == '\n' { + ans = append(ans, fmt.Sprintf(`"line": %d, `, linecnt)...) + // warning: this would fail if the spec file had + // `"value": {\n}`, but it does not, as comma is a separator. + } + case '\n': + linecnt++ + } + } + return ans +} + +// Type.Value has to be treated specially for literals and maps +func (t *Type) UnmarshalJSON(data []byte) error { + // First unmarshal only the unambiguous fields. + var x struct { + Kind string `json:"kind"` + Items []*Type `json:"items"` + Element *Type `json:"element"` + Name string `json:"name"` + Key *Type `json:"key"` + Value any `json:"value"` + Line int `json:"line"` + } + if err := json.Unmarshal(data, &x); err != nil { + return err + } + *t = Type{ + Kind: x.Kind, + Items: x.Items, + Element: x.Element, + Name: x.Name, + Value: x.Value, + Line: x.Line, + } + + // Then unmarshal the 'value' field based on the kind. + // This depends on Unmarshal ignoring fields it doesn't know about. + switch x.Kind { + case "map": + var x struct { + Key *Type `json:"key"` + Value *Type `json:"value"` + } + if err := json.Unmarshal(data, &x); err != nil { + return fmt.Errorf("Type.kind=map: %v", err) + } + t.Key = x.Key + t.Value = x.Value + + case "literal": + var z struct { + Value ParseLiteral `json:"value"` + } + + if err := json.Unmarshal(data, &z); err != nil { + return fmt.Errorf("Type.kind=literal: %v", err) + } + t.Value = z.Value + + case "base", "reference", "array", "and", "or", "tuple", + "stringLiteral": + // nop. never seen integerLiteral or booleanLiteral. + + default: + return fmt.Errorf("cannot decode Type.kind %q: %s", x.Kind, data) + } + return nil +} diff --git a/gopls/internal/lsp/protocol/generate/types.go b/gopls/internal/lsp/protocol/generate/types.go new file mode 100644 index 00000000000..e8abd600815 --- /dev/null +++ b/gopls/internal/lsp/protocol/generate/types.go @@ -0,0 +1,173 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.19 +// +build go1.19 + +package main + +import "sort" + +// Model contains the parsed version of the spec +type Model struct { + Version Metadata `json:"metaData"` + Requests []Request `json:"requests"` + Notifications []Notification `json:"notifications"` + Structures []Structure `json:"structures"` + Enumerations []Enumeration `json:"enumerations"` + TypeAliases []TypeAlias `json:"typeAliases"` + Line int `json:"line"` +} + +// Metadata is information about the version of the spec +type Metadata struct { + Version string `json:"version"` + Line int `json:"line"` +} + +// A Request is the parsed version of an LSP request +type Request struct { + Documentation string `json:"documentation"` + ErrorData *Type `json:"errorData"` + Direction string `json:"messageDirection"` + Method string `json:"method"` + Params *Type `json:"params"` + PartialResult *Type `json:"partialResult"` + Proposed bool `json:"proposed"` + RegistrationMethod string `json:"registrationMethod"` + RegistrationOptions *Type `json:"registrationOptions"` + Result *Type `json:"result"` + Since string `json:"since"` + Line int `json:"line"` +} + +// A Notificatin is the parsed version of an LSP notification +type Notification struct { + Documentation string `json:"documentation"` + Direction string `json:"messageDirection"` + Method string `json:"method"` + Params *Type `json:"params"` + Proposed bool `json:"proposed"` + RegistrationMethod string `json:"registrationMethod"` + RegistrationOptions *Type `json:"registrationOptions"` + Since string `json:"since"` + Line int `json:"line"` +} + +// A Structure is the parsed version of an LSP structure from the spec +type Structure struct { + Documentation string `json:"documentation"` + Extends []*Type `json:"extends"` + Mixins []*Type `json:"mixins"` + Name string `json:"name"` + Properties []NameType `json:"properties"` + Proposed bool `json:"proposed"` + Since string `json:"since"` + Line int `json:"line"` +} + +// An enumeration is the parsed version of an LSP enumeration from the spec +type Enumeration struct { + Documentation string `json:"documentation"` + Name string `json:"name"` + Proposed bool `json:"proposed"` + Since string `json:"since"` + SupportsCustomValues bool `json:"supportsCustomValues"` + Type *Type `json:"type"` + Values []NameValue `json:"values"` + Line int `json:"line"` +} + +// A TypeAlias is the parsed version of an LSP type alias from the spec +type TypeAlias struct { + Documentation string `json:"documentation"` + Name string `json:"name"` + Proposed bool `json:"proposed"` + Since string `json:"since"` + Type *Type `json:"type"` + Line int `json:"line"` +} + +// A NameValue describes an enumeration constant +type NameValue struct { + Documentation string `json:"documentation"` + Name string `json:"name"` + Proposed bool `json:"proposed"` + Since string `json:"since"` + Value any `json:"value"` // number or string + Line int `json:"line"` +} + +// common to Request and Notification +type Message interface { + direction() string +} + +func (r Request) direction() string { + return r.Direction +} + +func (n Notification) direction() string { + return n.Direction +} + +// A Defined is one of Structure, Enumeration, TypeAlias, for type checking +type Defined interface { + tag() +} + +func (s Structure) tag() { +} + +func (e Enumeration) tag() { +} + +func (ta TypeAlias) tag() { +} + +// A Type is the parsed version of an LSP type from the spec, +// or a Type the code constructs +type Type struct { + Kind string `json:"kind"` // -- which kind goes with which field -- + Items []*Type `json:"items"` // "and", "or", "tuple" + Element *Type `json:"element"` // "array" + Name string `json:"name"` // "base", "reference" + Key *Type `json:"key"` // "map" + Value any `json:"value"` // "map", "stringLiteral", "literal" + // used to tie generated code to the specification + Line int `json:"line"` + + name string // these are generated names, like Uint32 + typeName string // these are actual type names, like uint32 +} + +// ParsedLiteral is Type.Value when Type.Kind is "literal" +type ParseLiteral struct { + Properties `json:"properties"` +} + +// A NameType represents the name and type of a structure element +type NameType struct { + Name string `json:"name"` + Type *Type `json:"type"` + Optional bool `json:"optional"` + Documentation string `json:"documentation"` + Since string `json:"since"` + Proposed bool `json:"proposed"` + Line int `json:"line"` +} + +// Properties are the collection of structure elements +type Properties []NameType + +type sortedMap[T any] map[string]T + +func (s sortedMap[T]) keys() []string { + var keys []string + for k := range s { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/gopls/internal/lsp/protocol/generate/utilities.go b/gopls/internal/lsp/protocol/generate/utilities.go new file mode 100644 index 00000000000..b091a0d145f --- /dev/null +++ b/gopls/internal/lsp/protocol/generate/utilities.go @@ -0,0 +1,55 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.19 +// +build go1.19 + +package main + +import ( + "fmt" + "log" + "runtime" + "strings" + "time" +) + +// goName returns the Go version of a name. +func goName(s string) string { + if s == "" { + return s // doesn't happen + } + s = strings.ToUpper(s[:1]) + s[1:] + if rest := strings.TrimSuffix(s, "Uri"); rest != s { + s = rest + "URI" + } + if rest := strings.TrimSuffix(s, "Id"); rest != s { + s = rest + "ID" + } + return s +} + +// the common header for all generated files +func (s *spec) createHeader() string { + format := `// Copyright 2022 The Go Authors. All rights reserved. + // Use of this source code is governed by a BSD-style + // license that can be found in the LICENSE file. + + // Code generated for LSP. DO NOT EDIT. + + package protocol + + // Code generated from version %s of protocol/metaModel.json. + // git hash %s (as of %s) + + ` + hdr := fmt.Sprintf(format, s.model.Version.Version, s.githash, s.modTime.Format(time.ANSIC)) + return hdr +} + +// useful in debugging +func here() { + _, f, l, _ := runtime.Caller(1) + log.Printf("here: %s:%d", f, l) +} diff --git a/gopls/internal/lsp/regtest/expectation.go b/gopls/internal/lsp/regtest/expectation.go index 335a46fc589..d09398779c1 100644 --- a/gopls/internal/lsp/regtest/expectation.go +++ b/gopls/internal/lsp/regtest/expectation.go @@ -7,6 +7,7 @@ package regtest import ( "fmt" "regexp" + "sort" "strings" "golang.org/x/tools/gopls/internal/lsp" @@ -130,6 +131,33 @@ func AnyOf(anyOf ...Expectation) *SimpleExpectation { } } +// AllOf expects that all given expectations are met. +// +// TODO(rfindley): the problem with these types of combinators (OnceMet, AnyOf +// and AllOf) is that we lose the information of *why* they failed: the Awaiter +// is not smart enough to look inside. +// +// Refactor the API such that the Check function is responsible for explaining +// why an expectation failed. This should allow us to significantly improve +// test output: we won't need to summarize state at all, as the verdict +// explanation itself should describe clearly why the expectation not met. +func AllOf(allOf ...Expectation) *SimpleExpectation { + check := func(s State) Verdict { + verdict := Met + for _, e := range allOf { + if v := e.Check(s); v > verdict { + verdict = v + } + } + return verdict + } + description := describeExpectations(allOf...) + return &SimpleExpectation{ + check: check, + description: fmt.Sprintf("All of:\n%s", description), + } +} + // ReadDiagnostics is an 'expectation' that is used to read diagnostics // atomically. It is intended to be used with 'OnceMet'. func ReadDiagnostics(fileName string, into *protocol.PublishDiagnosticsParams) *SimpleExpectation { @@ -218,6 +246,54 @@ func ShowMessageRequest(title string) SimpleExpectation { } } +// DoneDiagnosingChanges expects that diagnostics are complete from common +// change notifications: didOpen, didChange, didSave, didChangeWatchedFiles, +// and didClose. +// +// This can be used when multiple notifications may have been sent, such as +// when a didChange is immediately followed by a didSave. It is insufficient to +// simply await NoOutstandingWork, because the LSP client has no control over +// when the server starts processing a notification. Therefore, we must keep +// track of +func (e *Env) DoneDiagnosingChanges() Expectation { + stats := e.Editor.Stats() + statsBySource := map[lsp.ModificationSource]uint64{ + lsp.FromDidOpen: stats.DidOpen, + lsp.FromDidChange: stats.DidChange, + lsp.FromDidSave: stats.DidSave, + lsp.FromDidChangeWatchedFiles: stats.DidChangeWatchedFiles, + lsp.FromDidClose: stats.DidClose, + } + + var expected []lsp.ModificationSource + for k, v := range statsBySource { + if v > 0 { + expected = append(expected, k) + } + } + + // Sort for stability. + sort.Slice(expected, func(i, j int) bool { + return expected[i] < expected[j] + }) + + var all []Expectation + for _, source := range expected { + all = append(all, CompletedWork(lsp.DiagnosticWorkTitle(source), statsBySource[source], true)) + } + + return AllOf(all...) +} + +// AfterChange expects that the given expectations will be met after all +// state-changing notifications have been processed by the server. +func (e *Env) AfterChange(expectations ...Expectation) Expectation { + return OnceMet( + e.DoneDiagnosingChanges(), + expectations..., + ) +} + // DoneWithOpen expects all didOpen notifications currently sent by the editor // to be completely processed. func (e *Env) DoneWithOpen() Expectation { diff --git a/gopls/internal/lsp/semantic.go b/gopls/internal/lsp/semantic.go index 713c3ac4c0f..a8b38f044ab 100644 --- a/gopls/internal/lsp/semantic.go +++ b/gopls/internal/lsp/semantic.go @@ -526,9 +526,14 @@ func (e *encoded) ident(x *ast.Ident) { tok(x.Pos(), len(x.Name), tokFunction, nil) } else if _, ok := y.Type().(*typeparams.TypeParam); ok { tok(x.Pos(), len(x.Name), tokTypeParam, nil) + } else if e.isParam(use.Pos()) { + // variable, unless use.pos is the pos of a Field in an ancestor FuncDecl + // or FuncLit and then it's a parameter + tok(x.Pos(), len(x.Name), tokParameter, nil) } else { tok(x.Pos(), len(x.Name), tokVariable, nil) } + default: // can't happen if use == nil { @@ -543,6 +548,30 @@ func (e *encoded) ident(x *ast.Ident) { } } +func (e *encoded) isParam(pos token.Pos) bool { + for i := len(e.stack) - 1; i >= 0; i-- { + switch n := e.stack[i].(type) { + case *ast.FuncDecl: + for _, f := range n.Type.Params.List { + for _, id := range f.Names { + if id.Pos() == pos { + return true + } + } + } + case *ast.FuncLit: + for _, f := range n.Type.Params.List { + for _, id := range f.Names { + if id.Pos() == pos { + return true + } + } + } + } + } + return false +} + func isSignature(use types.Object) bool { if true { return false //PJW: fix after generics seem ok @@ -640,7 +669,7 @@ func (e *encoded) unkIdent(x *ast.Ident) (tokenType, []string) { if nd.Tok != token.DEFINE { def = nil } - return tokVariable, def + return tokVariable, def // '_' in _ = ... } } // RHS, = x diff --git a/gopls/internal/lsp/source/api_json.go b/gopls/internal/lsp/source/api_json.go index 2b7362cb124..762054d10b8 100755 --- a/gopls/internal/lsp/source/api_json.go +++ b/gopls/internal/lsp/source/api_json.go @@ -307,7 +307,7 @@ var GeneratedAPIJSON = &APIJSON{ }, { Name: "\"loopclosure\"", - Doc: "check references to loop variables from within nested functions\n\nThis analyzer checks for references to loop variables from within a function\nliteral inside the loop body. It checks for patterns where access to a loop\nvariable is known to escape the current loop iteration:\n 1. a call to go or defer at the end of the loop body\n 2. a call to golang.org/x/sync/errgroup.Group.Go at the end of the loop body\n\nThe analyzer only considers references in the last statement of the loop body\nas it is not deep enough to understand the effects of subsequent statements\nwhich might render the reference benign.\n\nFor example:\n\n\tfor i, v := range s {\n\t\tgo func() {\n\t\t\tprintln(i, v) // not what you might expect\n\t\t}()\n\t}\n\nSee: https://golang.org/doc/go_faq.html#closures_and_goroutines", + Doc: "check references to loop variables from within nested functions\n\nThis analyzer checks for references to loop variables from within a function\nliteral inside the loop body. It checks for patterns where access to a loop\nvariable is known to escape the current loop iteration:\n 1. a call to go or defer at the end of the loop body\n 2. a call to golang.org/x/sync/errgroup.Group.Go at the end of the loop body\n 3. a call testing.T.Run where the subtest body invokes t.Parallel()\n\nIn the case of (1) and (2), the analyzer only considers references in the last\nstatement of the loop body as it is not deep enough to understand the effects\nof subsequent statements which might render the reference benign.\n\nFor example:\n\n\tfor i, v := range s {\n\t\tgo func() {\n\t\t\tprintln(i, v) // not what you might expect\n\t\t}()\n\t}\n\nSee: https://golang.org/doc/go_faq.html#closures_and_goroutines", Default: "true", }, { @@ -933,7 +933,7 @@ var GeneratedAPIJSON = &APIJSON{ }, { Name: "loopclosure", - Doc: "check references to loop variables from within nested functions\n\nThis analyzer checks for references to loop variables from within a function\nliteral inside the loop body. It checks for patterns where access to a loop\nvariable is known to escape the current loop iteration:\n 1. a call to go or defer at the end of the loop body\n 2. a call to golang.org/x/sync/errgroup.Group.Go at the end of the loop body\n\nThe analyzer only considers references in the last statement of the loop body\nas it is not deep enough to understand the effects of subsequent statements\nwhich might render the reference benign.\n\nFor example:\n\n\tfor i, v := range s {\n\t\tgo func() {\n\t\t\tprintln(i, v) // not what you might expect\n\t\t}()\n\t}\n\nSee: https://golang.org/doc/go_faq.html#closures_and_goroutines", + Doc: "check references to loop variables from within nested functions\n\nThis analyzer checks for references to loop variables from within a function\nliteral inside the loop body. It checks for patterns where access to a loop\nvariable is known to escape the current loop iteration:\n 1. a call to go or defer at the end of the loop body\n 2. a call to golang.org/x/sync/errgroup.Group.Go at the end of the loop body\n 3. a call testing.T.Run where the subtest body invokes t.Parallel()\n\nIn the case of (1) and (2), the analyzer only considers references in the last\nstatement of the loop body as it is not deep enough to understand the effects\nof subsequent statements which might render the reference benign.\n\nFor example:\n\n\tfor i, v := range s {\n\t\tgo func() {\n\t\t\tprintln(i, v) // not what you might expect\n\t\t}()\n\t}\n\nSee: https://golang.org/doc/go_faq.html#closures_and_goroutines", Default: true, }, { diff --git a/gopls/internal/lsp/source/completion/completion.go b/gopls/internal/lsp/source/completion/completion.go index 7d98205d020..c3b7c2b461b 100644 --- a/gopls/internal/lsp/source/completion/completion.go +++ b/gopls/internal/lsp/source/completion/completion.go @@ -1280,7 +1280,9 @@ func isStarTestingDotF(typ types.Type) bool { if named == nil { return false } - return named.Obj() != nil && named.Obj().Pkg().Path() == "testing" && named.Obj().Name() == "F" + obj := named.Obj() + // obj.Pkg is nil for the error type. + return obj != nil && obj.Pkg() != nil && obj.Pkg().Path() == "testing" && obj.Name() == "F" } // lexical finds completions in the lexical environment. diff --git a/gopls/internal/lsp/source/hover.go b/gopls/internal/lsp/source/hover.go index c758d688dd2..09f7224c80d 100644 --- a/gopls/internal/lsp/source/hover.go +++ b/gopls/internal/lsp/source/hover.go @@ -904,78 +904,80 @@ func FindDeclAndField(files []*ast.File, pos token.Pos) (decl ast.Decl, field *a }() // Visit the files in search of the node at pos. - var stack []ast.Node - for _, file := range files { - ast.Inspect(file, func(n ast.Node) bool { - if n != nil { - stack = append(stack, n) // push - } else { - stack = stack[:len(stack)-1] // pop - return false - } + stack := make([]ast.Node, 0, 20) + // Allocate the closure once, outside the loop. + f := func(n ast.Node) bool { + if n != nil { + stack = append(stack, n) // push + } else { + stack = stack[:len(stack)-1] // pop + return false + } - // Skip subtrees (incl. files) that don't contain the search point. - if !(n.Pos() <= pos && pos < n.End()) { - return false - } + // Skip subtrees (incl. files) that don't contain the search point. + if !(n.Pos() <= pos && pos < n.End()) { + return false + } - switch n := n.(type) { - case *ast.Field: - checkField := func(f ast.Node) { - if f.Pos() == pos { - field = n - for i := len(stack) - 1; i >= 0; i-- { - if d, ok := stack[i].(ast.Decl); ok { - decl = d // innermost enclosing decl - break - } + switch n := n.(type) { + case *ast.Field: + checkField := func(f ast.Node) { + if f.Pos() == pos { + field = n + for i := len(stack) - 1; i >= 0; i-- { + if d, ok := stack[i].(ast.Decl); ok { + decl = d // innermost enclosing decl + break } - panic(nil) // found } + panic(nil) // found } + } - // Check *ast.Field itself. This handles embedded - // fields which have no associated *ast.Ident name. - checkField(n) + // Check *ast.Field itself. This handles embedded + // fields which have no associated *ast.Ident name. + checkField(n) - // Check each field name since you can have - // multiple names for the same type expression. - for _, name := range n.Names { - checkField(name) - } + // Check each field name since you can have + // multiple names for the same type expression. + for _, name := range n.Names { + checkField(name) + } - // Also check "X" in "...X". This makes it easy - // to format variadic signature params properly. - if ell, ok := n.Type.(*ast.Ellipsis); ok && ell.Elt != nil { - checkField(ell.Elt) - } + // Also check "X" in "...X". This makes it easy + // to format variadic signature params properly. + if ell, ok := n.Type.(*ast.Ellipsis); ok && ell.Elt != nil { + checkField(ell.Elt) + } - case *ast.FuncDecl: - if n.Name.Pos() == pos { - decl = n - panic(nil) // found - } + case *ast.FuncDecl: + if n.Name.Pos() == pos { + decl = n + panic(nil) // found + } - case *ast.GenDecl: - for _, spec := range n.Specs { - switch spec := spec.(type) { - case *ast.TypeSpec: - if spec.Name.Pos() == pos { + case *ast.GenDecl: + for _, spec := range n.Specs { + switch spec := spec.(type) { + case *ast.TypeSpec: + if spec.Name.Pos() == pos { + decl = n + panic(nil) // found + } + case *ast.ValueSpec: + for _, id := range spec.Names { + if id.Pos() == pos { decl = n panic(nil) // found } - case *ast.ValueSpec: - for _, id := range spec.Names { - if id.Pos() == pos { - decl = n - panic(nil) // found - } - } } } } - return true - }) + } + return true + } + for _, file := range files { + ast.Inspect(file, f) } return nil, nil diff --git a/gopls/internal/lsp/source/options.go b/gopls/internal/lsp/source/options.go index 136ba23d86c..23d795ef0e5 100644 --- a/gopls/internal/lsp/source/options.go +++ b/gopls/internal/lsp/source/options.go @@ -613,11 +613,10 @@ type InternalOptions struct { // This option applies only during initialization. ShowBugReports bool - // NewDiff controls the choice of the new diff implementation. - // It can be 'new', 'checked', or 'old' which is the default. - // 'checked' computes diffs with both algorithms, checks - // that the new algorithm has worked, and write some summary - // statistics to a file in os.TmpDir() + // NewDiff controls the choice of the new diff implementation. It can be + // 'new', 'old', or 'both', which is the default. 'both' computes diffs with + // both algorithms, checks that the new algorithm has worked, and write some + // summary statistics to a file in os.TmpDir(). NewDiff string // ChattyDiagnostics controls whether to report file diagnostics for each @@ -845,8 +844,6 @@ func (o *Options) AddStaticcheckAnalyzer(a *analysis.Analyzer, enabled bool, sev // should be enabled in enableAllExperimentMaps. func (o *Options) EnableAllExperiments() { o.SemanticTokens = true - o.ExperimentalUseInvalidMetadata = true - o.ExperimentalWatchedFileDelay = 50 * time.Millisecond } func (o *Options) enableAllExperimentMaps() { @@ -1039,10 +1036,8 @@ func (o *Options) set(name string, value interface{}, seen map[string]struct{}) if v, ok := result.asBool(); ok { o.Staticcheck = v if v && !o.StaticcheckSupported { - // Warn if the user is trying to enable staticcheck, but staticcheck is - // unsupported. - result.Error = fmt.Errorf("applying setting %q: staticcheck is not supported at %s\n"+ - "\trebuild gopls with a more recent version of Go", result.Name, runtime.Version()) + result.Error = fmt.Errorf("applying setting %q: staticcheck is not supported at %s;"+ + " rebuild gopls with a more recent version of Go", result.Name, runtime.Version()) } } @@ -1062,7 +1057,13 @@ func (o *Options) set(name string, value interface{}, seen map[string]struct{}) result.setBool(&o.ShowBugReports) case "gofumpt": - result.setBool(&o.Gofumpt) + if v, ok := result.asBool(); ok { + o.Gofumpt = v + if v && o.GofumptFormat == nil { + result.Error = fmt.Errorf("applying setting %q: gofumpt is not supported at %s;"+ + " rebuild gopls with a more recent version of Go", result.Name, runtime.Version()) + } + } case "semanticTokens": result.setBool(&o.SemanticTokens) diff --git a/gopls/internal/lsp/source/workspace_symbol.go b/gopls/internal/lsp/source/workspace_symbol.go index bd1e7b12adc..ee4e020e257 100644 --- a/gopls/internal/lsp/source/workspace_symbol.go +++ b/gopls/internal/lsp/source/workspace_symbol.go @@ -379,6 +379,7 @@ func NewFilterer(rawFilters []string) *Filterer { var f Filterer for _, filter := range rawFilters { filter = path.Clean(filepath.ToSlash(filter)) + // TODO(dungtuanle): fix: validate [+-] prefix. op, prefix := filter[0], filter[1:] // convertFilterToRegexp adds "/" at the end of prefix to handle cases where a filter is a prefix of another filter. // For example, it prevents [+foobar, -foo] from excluding "foobar". @@ -391,20 +392,19 @@ func NewFilterer(rawFilters []string) *Filterer { // Disallow return true if the path is excluded from the filterer's filters. func (f *Filterer) Disallow(path string) bool { + // Ensure trailing but not leading slash. path = strings.TrimPrefix(path, "/") - var excluded bool + if !strings.HasSuffix(path, "/") { + path += "/" + } + // TODO(adonovan): opt: iterate in reverse and break at first match. + excluded := false for i, filter := range f.filters { - path := path - if !strings.HasSuffix(path, "/") { - path += "/" - } - if !filter.MatchString(path) { - continue + if filter.MatchString(path) { + excluded = f.excluded[i] // last match wins } - excluded = f.excluded[i] } - return excluded } @@ -419,6 +419,7 @@ func convertFilterToRegexp(filter string) *regexp.Regexp { ret.WriteString("^") segs := strings.Split(filter, "/") for _, seg := range segs { + // Inv: seg != "" since path is clean. if seg == "**" { ret.WriteString(".*") } else { @@ -426,8 +427,15 @@ func convertFilterToRegexp(filter string) *regexp.Regexp { } ret.WriteString("/") } + pattern := ret.String() + + // Remove unnecessary "^.*" prefix, which increased + // BenchmarkWorkspaceSymbols time by ~20% (even though + // filter CPU time increased by only by ~2.5%) when the + // default filter was changed to "**/node_modules". + pattern = strings.TrimPrefix(pattern, "^.*") - return regexp.MustCompile(ret.String()) + return regexp.MustCompile(pattern) } // symbolFile holds symbol information for a single file. diff --git a/gopls/internal/lsp/testdata/issues/issue56505.go b/gopls/internal/lsp/testdata/issues/issue56505.go new file mode 100644 index 00000000000..8c641bfb852 --- /dev/null +++ b/gopls/internal/lsp/testdata/issues/issue56505.go @@ -0,0 +1,8 @@ +package issues + +// Test for golang/go#56505: completion on variables of type *error should not +// panic. +func _() { + var e *error + e.x //@complete(" //") +} diff --git a/gopls/internal/lsp/testdata/semantic/a.go.golden b/gopls/internal/lsp/testdata/semantic/a.go.golden index 071dd171c84..34b70e0f4f2 100644 --- a/gopls/internal/lsp/testdata/semantic/a.go.golden +++ b/gopls/internal/lsp/testdata/semantic/a.go.golden @@ -65,7 +65,7 @@ /*⇒2,variable,[definition]*/ff /*⇒2,operator,[]*/:= /*⇒4,keyword,[]*/func() {} /*⇒5,keyword,[]*/defer /*⇒2,variable,[]*/ff() /*⇒2,keyword,[]*/go /*⇒3,namespace,[]*/utf./*⇒9,function,[]*/RuneCount(/*⇒2,string,[]*/"") - /*⇒2,keyword,[]*/go /*⇒4,namespace,[]*/utf8./*⇒9,function,[]*/RuneCount(/*⇒2,variable,[]*/vv.(/*⇒6,type,[]*/string)) + /*⇒2,keyword,[]*/go /*⇒4,namespace,[]*/utf8./*⇒9,function,[]*/RuneCount(/*⇒2,parameter,[]*/vv.(/*⇒6,type,[]*/string)) /*⇒2,keyword,[]*/if /*⇒4,variable,[readonly]*/true { } /*⇒4,keyword,[]*/else { } @@ -73,9 +73,9 @@ /*⇒3,keyword,[]*/for /*⇒1,variable,[definition]*/i /*⇒2,operator,[]*/:= /*⇒1,number,[]*/0; /*⇒1,variable,[]*/i /*⇒1,operator,[]*/< /*⇒2,number,[]*/10; { /*⇒5,keyword,[]*/break Never } - _, /*⇒2,variable,[definition]*/ok /*⇒2,operator,[]*/:= /*⇒2,variable,[]*/vv[/*⇒1,number,[]*/0].(/*⇒1,type,[]*/A) + _, /*⇒2,variable,[definition]*/ok /*⇒2,operator,[]*/:= /*⇒2,parameter,[]*/vv[/*⇒1,number,[]*/0].(/*⇒1,type,[]*/A) /*⇒2,keyword,[]*/if /*⇒1,operator,[]*/!/*⇒2,variable,[]*/ok { - /*⇒6,keyword,[]*/switch /*⇒1,variable,[definition]*/x /*⇒2,operator,[]*/:= /*⇒2,variable,[]*/vv[/*⇒1,number,[]*/0].(/*⇒4,keyword,[]*/type) { + /*⇒6,keyword,[]*/switch /*⇒1,variable,[definition]*/x /*⇒2,operator,[]*/:= /*⇒2,parameter,[]*/vv[/*⇒1,number,[]*/0].(/*⇒4,keyword,[]*/type) { } /*⇒4,keyword,[]*/goto Never } diff --git a/gopls/internal/lsp/testdata/summary.txt.golden b/gopls/internal/lsp/testdata/summary.txt.golden index 107e900a731..cfe8e4a267d 100644 --- a/gopls/internal/lsp/testdata/summary.txt.golden +++ b/gopls/internal/lsp/testdata/summary.txt.golden @@ -1,7 +1,7 @@ -- summary -- CallHierarchyCount = 2 CodeLensCount = 5 -CompletionsCount = 262 +CompletionsCount = 263 CompletionSnippetCount = 106 UnimportedCompletionsCount = 5 DeepCompletionsCount = 5 diff --git a/gopls/internal/lsp/testdata/summary_go1.18.txt.golden b/gopls/internal/lsp/testdata/summary_go1.18.txt.golden index 1e1c8762a8f..2b7bf976b2f 100644 --- a/gopls/internal/lsp/testdata/summary_go1.18.txt.golden +++ b/gopls/internal/lsp/testdata/summary_go1.18.txt.golden @@ -1,7 +1,7 @@ -- summary -- CallHierarchyCount = 2 CodeLensCount = 5 -CompletionsCount = 263 +CompletionsCount = 264 CompletionSnippetCount = 115 UnimportedCompletionsCount = 5 DeepCompletionsCount = 5 diff --git a/gopls/internal/lsp/text_synchronization.go b/gopls/internal/lsp/text_synchronization.go index 63bc0e8e561..ab765b60dd3 100644 --- a/gopls/internal/lsp/text_synchronization.go +++ b/gopls/internal/lsp/text_synchronization.go @@ -40,6 +40,9 @@ const ( // FromDidClose is a file modification caused by closing a file. FromDidClose + // TODO: add FromDidChangeConfiguration, once configuration changes cause a + // new snapshot to be created. + // FromRegenerateCgo refers to file modifications caused by regenerating // the cgo sources for the workspace. FromRegenerateCgo diff --git a/gopls/internal/regtest/diagnostics/diagnostics_test.go b/gopls/internal/regtest/diagnostics/diagnostics_test.go index f0082073326..18c022cb2a5 100644 --- a/gopls/internal/regtest/diagnostics/diagnostics_test.go +++ b/gopls/internal/regtest/diagnostics/diagnostics_test.go @@ -1358,7 +1358,7 @@ func _() { func TestEnableAllExperiments(t *testing.T) { // Before the oldest supported Go version, gopls sends a warning to upgrade // Go, which fails the expectation below. - testenv.NeedsGo1Point(t, lsp.OldestSupportedGoVersion) + testenv.NeedsGo1Point(t, lsp.OldestSupportedGoVersion()) const mod = ` -- go.mod -- diff --git a/gopls/internal/regtest/misc/configuration_test.go b/gopls/internal/regtest/misc/configuration_test.go index a1d46f036c6..5bb2c8620a0 100644 --- a/gopls/internal/regtest/misc/configuration_test.go +++ b/gopls/internal/regtest/misc/configuration_test.go @@ -80,6 +80,21 @@ var FooErr = errors.New("foo") }) } +func TestGofumptWarning(t *testing.T) { + testenv.SkipAfterGo1Point(t, 17) + + WithOptions( + Settings{"gofumpt": true}, + ).Run(t, "", func(t *testing.T, env *Env) { + env.Await( + OnceMet( + InitialWorkspaceLoad, + ShownMessage("gofumpt is not supported"), + ), + ) + }) +} + func TestDeprecatedSettings(t *testing.T) { WithOptions( Settings{ diff --git a/gopls/internal/regtest/misc/formatting_test.go b/gopls/internal/regtest/misc/formatting_test.go index 6b20afa98f0..39d58229896 100644 --- a/gopls/internal/regtest/misc/formatting_test.go +++ b/gopls/internal/regtest/misc/formatting_test.go @@ -10,6 +10,7 @@ import ( . "golang.org/x/tools/gopls/internal/lsp/regtest" "golang.org/x/tools/gopls/internal/lsp/tests/compare" + "golang.org/x/tools/internal/testenv" ) const unformattedProgram = ` @@ -302,6 +303,7 @@ func main() { } func TestGofumptFormatting(t *testing.T) { + testenv.NeedsGo1Point(t, 18) // Exercise some gofumpt formatting rules: // - No empty lines following an assignment operator diff --git a/gopls/internal/regtest/misc/vuln_test.go b/gopls/internal/regtest/misc/vuln_test.go index ecebfb52202..2afef9df78a 100644 --- a/gopls/internal/regtest/misc/vuln_test.go +++ b/gopls/internal/regtest/misc/vuln_test.go @@ -9,6 +9,7 @@ package misc import ( "context" + "strings" "testing" "golang.org/x/tools/gopls/internal/lsp/command" @@ -341,40 +342,36 @@ func TestRunVulncheckExp(t *testing.T) { // codeActions is a list titles of code actions that we get with context // diagnostics. codeActions []string + // hover message is the list of expected hover message parts for this go.mod require line. + // all parts must appear in the hover message. + hover []string }{ "golang.org/amod": { applyAction: "Upgrade to v1.0.4", diagnostics: []diagnostic{ { - msg: "golang.org/amod has a known vulnerability: vuln in amod", + msg: "golang.org/amod has a vulnerability used in the code: GO-2022-01.", severity: protocol.SeverityWarning, codeActions: []string{ "Upgrade to latest", "Upgrade to v1.0.4", }, }, - { - msg: "golang.org/amod has a known vulnerability: unaffecting vulnerability", - severity: protocol.SeverityInformation, - codeActions: []string{ - "Upgrade to latest", - "Upgrade to v1.0.6", - }, - }, }, codeActions: []string{ "Upgrade to latest", - "Upgrade to v1.0.6", "Upgrade to v1.0.4", }, + hover: []string{"GO-2022-01", "Fixed in v1.0.4.", "GO-2022-03"}, }, "golang.org/bmod": { diagnostics: []diagnostic{ { - msg: "golang.org/bmod has a known vulnerability: vuln in bmod\n\nThis is a long description of this vulnerability.", + msg: "golang.org/bmod has a vulnerability used in the code: GO-2022-02.", severity: protocol.SeverityWarning, }, }, + hover: []string{"GO-2022-02", "This is a long description of this vulnerability.", "No fix is available."}, }, } @@ -405,6 +402,17 @@ func TestRunVulncheckExp(t *testing.T) { t.Errorf("code actions for %q do not match, expected %v, got %v\n", w.msg, w.codeActions, gotActions) continue } + + // Check that useful info is supplemented as hover. + if len(want.hover) > 0 { + hover, _ := env.Hover("go.mod", pos) + for _, part := range want.hover { + if !strings.Contains(hover.Value, part) { + t.Errorf("hover contents for %q do not match, expected %v, got %v\n", w.msg, strings.Join(want.hover, ","), hover.Value) + break + } + } + } } // Check that the actions we get when including all diagnostics at a location return the same result diff --git a/gopls/internal/regtest/modfile/modfile_test.go b/gopls/internal/regtest/modfile/modfile_test.go index eb3f9665696..64892be5966 100644 --- a/gopls/internal/regtest/modfile/modfile_test.go +++ b/gopls/internal/regtest/modfile/modfile_test.go @@ -633,7 +633,7 @@ func main() { d := protocol.PublishDiagnosticsParams{} env.Await( - OnceMet( + env.AfterChange( // Make sure the diagnostic mentions the new version -- the old diagnostic is in the same place. env.DiagnosticAtRegexpWithMessage("a/go.mod", "example.com v1.2.3", "example.com@v1.2.3"), ReadDiagnostics("a/go.mod", &d), @@ -646,8 +646,10 @@ func main() { env.ApplyCodeAction(qfs[0]) // Arbitrarily pick a single fix to apply. Applying all of them seems to cause trouble in this particular test. env.SaveBuffer("a/go.mod") // Save to trigger diagnostics. env.Await( - EmptyDiagnostics("a/go.mod"), - env.DiagnosticAtRegexp("a/main.go", "x = "), + env.AfterChange( + EmptyDiagnostics("a/go.mod"), + env.DiagnosticAtRegexp("a/main.go", "x = "), + ), ) }) }) @@ -677,17 +679,23 @@ func main() { runner.Run(t, known, func(t *testing.T, env *Env) { env.OpenFile("a/go.mod") env.Await( - env.DiagnosticAtRegexp("a/main.go", "x = "), + env.AfterChange( + env.DiagnosticAtRegexp("a/main.go", "x = "), + ), ) env.RegexpReplace("a/go.mod", "v1.2.3", "v1.2.2") env.Editor.SaveBuffer(env.Ctx, "a/go.mod") // go.mod changes must be on disk env.Await( - env.DiagnosticAtRegexp("a/go.mod", "example.com v1.2.2"), + env.AfterChange( + env.DiagnosticAtRegexp("a/go.mod", "example.com v1.2.2"), + ), ) env.RegexpReplace("a/go.mod", "v1.2.2", "v1.2.3") env.Editor.SaveBuffer(env.Ctx, "a/go.mod") // go.mod changes must be on disk env.Await( - env.DiagnosticAtRegexp("a/main.go", "x = "), + env.AfterChange( + env.DiagnosticAtRegexp("a/main.go", "x = "), + ), ) }) }) diff --git a/gopls/internal/regtest/workspace/workspace_test.go b/gopls/internal/regtest/workspace/workspace_test.go index a271bb0fa7a..5786f0a031b 100644 --- a/gopls/internal/regtest/workspace/workspace_test.go +++ b/gopls/internal/regtest/workspace/workspace_test.go @@ -35,6 +35,8 @@ go 1.12 -- example.com@v1.2.3/blah/blah.go -- package blah +import "fmt" + func SaySomething() { fmt.Println("something") } @@ -62,7 +64,7 @@ require ( random.org v1.2.3 ) -- pkg/go.sum -- -example.com v1.2.3 h1:Yryq11hF02fEf2JlOS2eph+ICE2/ceevGV3C9dl5V/c= +example.com v1.2.3 h1:veRD4tUnatQRgsULqULZPjeoBGFr2qBhevSCZllD2Ds= example.com v1.2.3/go.mod h1:Y2Rc5rVWjWur0h3pd9aEvK5Pof8YKDANh9gHA2Maujo= random.org v1.2.3 h1:+JE2Fkp7gS0zsHXGEQJ7hraom3pNTlkxC4b2qPfA+/Q= random.org v1.2.3/go.mod h1:E9KM6+bBX2g5ykHZ9H27w16sWo3QwgonyjM44Dnej3I= @@ -216,6 +218,8 @@ go 1.12 -- example.com@v1.2.3/blah/blah.go -- package blah +import "fmt" + func SaySomething() { fmt.Println("something") } @@ -284,6 +288,8 @@ require b.com v1.2.3 -- c.com@v1.2.3/blah/blah.go -- package blah +import "fmt" + func SaySomething() { fmt.Println("something") } @@ -329,7 +335,12 @@ func main() { // This change tests that the version of the module used changes after it has // been deleted from the workspace. +// +// TODO(golang/go#55331): delete this placeholder along with experimental +// workspace module. func TestDeleteModule_Interdependent(t *testing.T) { + t.Skip("golang/go#55331: the experimental workspace module is scheduled for deletion") + const multiModule = ` -- moda/a/go.mod -- module a.com @@ -518,7 +529,7 @@ module b.com require example.com v1.2.3 -- modb/go.sum -- -example.com v1.2.3 h1:Yryq11hF02fEf2JlOS2eph+ICE2/ceevGV3C9dl5V/c= +example.com v1.2.3 h1:veRD4tUnatQRgsULqULZPjeoBGFr2qBhevSCZllD2Ds= example.com v1.2.3/go.mod h1:Y2Rc5rVWjWur0h3pd9aEvK5Pof8YKDANh9gHA2Maujo= -- modb/b/b.go -- package b @@ -1242,7 +1253,7 @@ import ( // supported. func TestOldGoNotification_SupportedVersion(t *testing.T) { v := goVersion(t) - if v < lsp.OldestSupportedGoVersion { + if v < lsp.OldestSupportedGoVersion() { t.Skipf("go version 1.%d is unsupported", v) } @@ -1261,7 +1272,7 @@ func TestOldGoNotification_SupportedVersion(t *testing.T) { // legacy Go versions (see also TestOldGoNotification_Fake) func TestOldGoNotification_UnsupportedVersion(t *testing.T) { v := goVersion(t) - if v >= lsp.OldestSupportedGoVersion { + if v >= lsp.OldestSupportedGoVersion() { t.Skipf("go version 1.%d is supported", v) } @@ -1285,10 +1296,12 @@ func TestOldGoNotification_Fake(t *testing.T) { if err != nil { t.Fatal(err) } - defer func(v int) { - lsp.OldestSupportedGoVersion = v - }(lsp.OldestSupportedGoVersion) - lsp.OldestSupportedGoVersion = goversion + 1 + defer func(t []lsp.GoVersionSupport) { + lsp.GoVersionTable = t + }(lsp.GoVersionTable) + lsp.GoVersionTable = []lsp.GoVersionSupport{ + {GoVersion: goversion, InstallGoplsVersion: "v1.0.0"}, + } Run(t, "", func(t *testing.T, env *Env) { env.Await( diff --git a/internal/analysisinternal/analysis.go b/internal/analysisinternal/analysis.go index 3b983ccf7d8..6fceef5e720 100644 --- a/internal/analysisinternal/analysis.go +++ b/internal/analysisinternal/analysis.go @@ -18,10 +18,6 @@ import ( // in Go 1.18+. var DiagnoseFuzzTests bool = false -// LoopclosureParallelSubtests controls whether the 'loopclosure' analyzer -// diagnoses loop variables references in parallel subtests. -var LoopclosureParallelSubtests = false - var ( GetTypeErrors func(p interface{}) []types.Error SetTypeErrors func(p interface{}, errors []types.Error) diff --git a/go/analysis/internal/facts/facts.go b/internal/facts/facts.go similarity index 91% rename from go/analysis/internal/facts/facts.go rename to internal/facts/facts.go index 006abab84ef..81df45161a8 100644 --- a/go/analysis/internal/facts/facts.go +++ b/internal/facts/facts.go @@ -152,6 +152,23 @@ type gobFact struct { Fact analysis.Fact // type and value of user-defined Fact } +// A Decoder decodes the facts from the direct imports of the package +// provided to NewEncoder. A single decoder may be used to decode +// multiple fact sets (e.g. each for a different set of fact types) +// for the same package. Each call to Decode returns an independent +// fact set. +type Decoder struct { + pkg *types.Package + packages map[string]*types.Package +} + +// NewDecoder returns a fact decoder for the specified package. +func NewDecoder(pkg *types.Package) *Decoder { + // Compute the import map for this package. + // See the package doc comment. + return &Decoder{pkg, importMap(pkg.Imports())} +} + // Decode decodes all the facts relevant to the analysis of package pkg. // The read function reads serialized fact data from an external source // for one of of pkg's direct imports. The empty file is a valid @@ -159,28 +176,24 @@ type gobFact struct { // // It is the caller's responsibility to call gob.Register on all // necessary fact types. -func Decode(pkg *types.Package, read func(packagePath string) ([]byte, error)) (*Set, error) { - // Compute the import map for this package. - // See the package doc comment. - packages := importMap(pkg.Imports()) - +func (d *Decoder) Decode(read func(*types.Package) ([]byte, error)) (*Set, error) { // Read facts from imported packages. // Facts may describe indirectly imported packages, or their objects. m := make(map[key]analysis.Fact) // one big bucket - for _, imp := range pkg.Imports() { + for _, imp := range d.pkg.Imports() { logf := func(format string, args ...interface{}) { if debug { prefix := fmt.Sprintf("in %s, importing %s: ", - pkg.Path(), imp.Path()) + d.pkg.Path(), imp.Path()) log.Print(prefix, fmt.Sprintf(format, args...)) } } // Read the gob-encoded facts. - data, err := read(imp.Path()) + data, err := read(imp) if err != nil { return nil, fmt.Errorf("in %s, can't import facts for package %q: %v", - pkg.Path(), imp.Path(), err) + d.pkg.Path(), imp.Path(), err) } if len(data) == 0 { continue // no facts @@ -195,7 +208,7 @@ func Decode(pkg *types.Package, read func(packagePath string) ([]byte, error)) ( // Parse each one into a key and a Fact. for _, f := range gobFacts { - factPkg := packages[f.PkgPath] + factPkg := d.packages[f.PkgPath] if factPkg == nil { // Fact relates to a dependency that was // unused in this translation unit. Skip. @@ -222,7 +235,7 @@ func Decode(pkg *types.Package, read func(packagePath string) ([]byte, error)) ( } } - return &Set{pkg: pkg, m: m}, nil + return &Set{pkg: d.pkg, m: m}, nil } // Encode encodes a set of facts to a memory buffer. diff --git a/go/analysis/internal/facts/facts_test.go b/internal/facts/facts_test.go similarity index 96% rename from go/analysis/internal/facts/facts_test.go rename to internal/facts/facts_test.go index c8379c58aa8..5c7b12ef1d4 100644 --- a/go/analysis/internal/facts/facts_test.go +++ b/internal/facts/facts_test.go @@ -14,8 +14,8 @@ import ( "testing" "golang.org/x/tools/go/analysis/analysistest" - "golang.org/x/tools/go/analysis/internal/facts" "golang.org/x/tools/go/packages" + "golang.org/x/tools/internal/facts" "golang.org/x/tools/internal/testenv" "golang.org/x/tools/internal/typeparams" ) @@ -216,7 +216,7 @@ type pkgLookups struct { // are passed during analysis. It operates on a group of Go file contents. Then // for each in tests it does the following: // 1. loads and type checks the package, -// 2. calls facts.Decode to loads the facts exported by its imports, +// 2. calls (*facts.Decoder).Decode to load the facts exported by its imports, // 3. exports a myFact Fact for all of package level objects, // 4. For each lookup for the current package: // 4.a) lookup the types.Object for an Go source expression in the curent package @@ -239,7 +239,7 @@ func testEncodeDecode(t *testing.T, files map[string]string, tests []pkgLookups) // factmap represents the passing of encoded facts from one // package to another. In practice one would use the file system. factmap := make(map[string][]byte) - read := func(path string) ([]byte, error) { return factmap[path], nil } + read := func(imp *types.Package) ([]byte, error) { return factmap[imp.Path()], nil } // Analyze packages in order, look up various objects accessible within // each package, and see if they have a fact. The "analysis" exports a @@ -255,7 +255,7 @@ func testEncodeDecode(t *testing.T, files map[string]string, tests []pkgLookups) } // decode - facts, err := facts.Decode(pkg, read) + facts, err := facts.NewDecoder(pkg).Decode(read) if err != nil { t.Fatalf("Decode failed: %v", err) } @@ -357,7 +357,7 @@ func TestFactFilter(t *testing.T) { } obj := pkg.Scope().Lookup("A") - s, err := facts.Decode(pkg, func(string) ([]byte, error) { return nil, nil }) + s, err := facts.NewDecoder(pkg).Decode(func(*types.Package) ([]byte, error) { return nil, nil }) if err != nil { t.Fatal(err) } diff --git a/go/analysis/internal/facts/imports.go b/internal/facts/imports.go similarity index 95% rename from go/analysis/internal/facts/imports.go rename to internal/facts/imports.go index 8a5553e2e9b..a3aa90dd1c5 100644 --- a/go/analysis/internal/facts/imports.go +++ b/internal/facts/imports.go @@ -20,6 +20,9 @@ import ( // // Packages in the map that are only indirectly imported may be // incomplete (!pkg.Complete()). +// +// TODO(adonovan): opt: compute this information more efficiently +// by obtaining it from the internals of the gcexportdata decoder. func importMap(imports []*types.Package) map[string]*types.Package { objects := make(map[types.Object]bool) packages := make(map[string]*types.Package) diff --git a/go/internal/gcimporter/bexport.go b/internal/gcimporter/bexport.go similarity index 99% rename from go/internal/gcimporter/bexport.go rename to internal/gcimporter/bexport.go index 196cb3f9b41..30582ed6d3d 100644 --- a/go/internal/gcimporter/bexport.go +++ b/internal/gcimporter/bexport.go @@ -12,7 +12,6 @@ import ( "bytes" "encoding/binary" "fmt" - "go/ast" "go/constant" "go/token" "go/types" @@ -145,7 +144,7 @@ func BExportData(fset *token.FileSet, pkg *types.Package) (b []byte, err error) objcount := 0 scope := pkg.Scope() for _, name := range scope.Names() { - if !ast.IsExported(name) { + if !token.IsExported(name) { continue } if trace { @@ -482,7 +481,7 @@ func (p *exporter) method(m *types.Func) { p.pos(m) p.string(m.Name()) - if m.Name() != "_" && !ast.IsExported(m.Name()) { + if m.Name() != "_" && !token.IsExported(m.Name()) { p.pkg(m.Pkg(), false) } @@ -501,7 +500,7 @@ func (p *exporter) fieldName(f *types.Var) { // 3) field name doesn't match base type name (alias name) bname := basetypeName(f.Type()) if name == bname { - if ast.IsExported(name) { + if token.IsExported(name) { name = "" // 1) we don't need to know the field name or package } else { name = "?" // 2) use unexported name "?" to force package export @@ -514,7 +513,7 @@ func (p *exporter) fieldName(f *types.Var) { } p.string(name) - if name != "" && !ast.IsExported(name) { + if name != "" && !token.IsExported(name) { p.pkg(f.Pkg(), false) } } diff --git a/go/internal/gcimporter/bexport_test.go b/internal/gcimporter/bexport_test.go similarity index 99% rename from go/internal/gcimporter/bexport_test.go rename to internal/gcimporter/bexport_test.go index 3da5397eb50..b5e9ce10044 100644 --- a/go/internal/gcimporter/bexport_test.go +++ b/internal/gcimporter/bexport_test.go @@ -21,8 +21,8 @@ import ( "golang.org/x/tools/go/ast/inspector" "golang.org/x/tools/go/buildutil" - "golang.org/x/tools/go/internal/gcimporter" "golang.org/x/tools/go/loader" + "golang.org/x/tools/internal/gcimporter" "golang.org/x/tools/internal/typeparams" "golang.org/x/tools/internal/typeparams/genericfeatures" ) @@ -109,7 +109,7 @@ type UnknownType undefined // Compare the packages' corresponding members. for _, name := range pkg.Scope().Names() { - if !ast.IsExported(name) { + if !token.IsExported(name) { continue } obj1 := pkg.Scope().Lookup(name) diff --git a/go/internal/gcimporter/bimport.go b/internal/gcimporter/bimport.go similarity index 100% rename from go/internal/gcimporter/bimport.go rename to internal/gcimporter/bimport.go diff --git a/go/internal/gcimporter/exportdata.go b/internal/gcimporter/exportdata.go similarity index 100% rename from go/internal/gcimporter/exportdata.go rename to internal/gcimporter/exportdata.go diff --git a/go/internal/gcimporter/gcimporter.go b/internal/gcimporter/gcimporter.go similarity index 96% rename from go/internal/gcimporter/gcimporter.go rename to internal/gcimporter/gcimporter.go index e96c39600d1..f8369cdc52e 100644 --- a/go/internal/gcimporter/gcimporter.go +++ b/internal/gcimporter/gcimporter.go @@ -9,7 +9,7 @@ // Package gcimporter provides various functions for reading // gc-generated object files that can be used to implement the // Importer interface defined by the Go 1.5 standard library package. -package gcimporter // import "golang.org/x/tools/go/internal/gcimporter" +package gcimporter // import "golang.org/x/tools/internal/gcimporter" import ( "bufio" @@ -22,11 +22,14 @@ import ( "io" "io/ioutil" "os" + "path" "path/filepath" "sort" "strconv" "strings" "text/scanner" + + "golang.org/x/tools/internal/goroot" ) const ( @@ -38,6 +41,25 @@ const ( trace = false ) +func lookupGorootExport(pkgpath, srcRoot, srcDir string) (string, bool) { + pkgpath = filepath.ToSlash(pkgpath) + m, err := goroot.PkgfileMap() + if err != nil { + return "", false + } + if export, ok := m[pkgpath]; ok { + return export, true + } + vendorPrefix := "vendor" + if strings.HasPrefix(srcDir, filepath.Join(srcRoot, "cmd")) { + vendorPrefix = path.Join("cmd", vendorPrefix) + } + pkgpath = path.Join(vendorPrefix, pkgpath) + fmt.Fprintln(os.Stderr, "looking up ", pkgpath) + export, ok := m[pkgpath] + return export, ok +} + var pkgExts = [...]string{".a", ".o"} // FindPkg returns the filename and unique package id for an import @@ -60,11 +82,18 @@ func FindPkg(path, srcDir string) (filename, id string) { } bp, _ := build.Import(path, srcDir, build.FindOnly|build.AllowBinary) if bp.PkgObj == "" { - id = path // make sure we have an id to print in error message - return + var ok bool + if bp.Goroot { + filename, ok = lookupGorootExport(path, bp.SrcRoot, srcDir) + } + if !ok { + id = path // make sure we have an id to print in error message + return + } + } else { + noext = strings.TrimSuffix(bp.PkgObj, ".a") + id = bp.ImportPath } - noext = strings.TrimSuffix(bp.PkgObj, ".a") - id = bp.ImportPath case build.IsLocalImport(path): // "./x" -> "/this/directory/x.ext", "/this/directory/x" @@ -85,6 +114,12 @@ func FindPkg(path, srcDir string) (filename, id string) { } } + if filename != "" { + if f, err := os.Stat(filename); err == nil && !f.IsDir() { + return + } + } + // try extensions for _, ext := range pkgExts { filename = noext + ext diff --git a/go/internal/gcimporter/gcimporter_test.go b/internal/gcimporter/gcimporter_test.go similarity index 79% rename from go/internal/gcimporter/gcimporter_test.go rename to internal/gcimporter/gcimporter_test.go index 5e1ca4bebcc..e4029c0d5e1 100644 --- a/go/internal/gcimporter/gcimporter_test.go +++ b/internal/gcimporter/gcimporter_test.go @@ -10,8 +10,12 @@ package gcimporter import ( "bytes" "fmt" + "go/ast" "go/build" "go/constant" + goimporter "go/importer" + goparser "go/parser" + "go/token" "go/types" "io/ioutil" "os" @@ -44,25 +48,31 @@ func needsCompiler(t *testing.T, compiler string) { // compile runs the compiler on filename, with dirname as the working directory, // and writes the output file to outdirname. -func compile(t *testing.T, dirname, filename, outdirname string) string { - return compilePkg(t, dirname, filename, outdirname, "p") +// compile gives the resulting package a packagepath of p. +func compile(t *testing.T, dirname, filename, outdirname string, packagefiles map[string]string) string { + return compilePkg(t, dirname, filename, outdirname, packagefiles, "p") } -func compilePkg(t *testing.T, dirname, filename, outdirname, pkg string) string { +func compilePkg(t *testing.T, dirname, filename, outdirname string, packagefiles map[string]string, pkg string) string { testenv.NeedsGoBuild(t) // filename must end with ".go" - if !strings.HasSuffix(filename, ".go") { + basename := strings.TrimSuffix(filepath.Base(filename), ".go") + ok := filename != basename + if !ok { t.Fatalf("filename doesn't end in .go: %s", filename) } - basename := filepath.Base(filename) - outname := filepath.Join(outdirname, basename[:len(basename)-2]+"o") - cmd := exec.Command("go", "tool", "compile", "-p="+pkg, "-o", outname, filename) + objname := basename + ".o" + outname := filepath.Join(outdirname, objname) + importcfgfile := filepath.Join(outdirname, basename) + ".importcfg" + testenv.WriteImportcfg(t, importcfgfile, packagefiles) + importreldir := strings.ReplaceAll(outdirname, string(os.PathSeparator), "/") + cmd := exec.Command("go", "tool", "compile", "-p", pkg, "-D", importreldir, "-importcfg", importcfgfile, "-o", outname, filename) cmd.Dir = dirname out, err := cmd.CombinedOutput() if err != nil { t.Logf("%s", out) - t.Fatalf("(cd %v && %v) failed: %s", cmd.Dir, cmd, err) + t.Fatalf("go tool compile %s failed: %s", filename, err) } return outname } @@ -129,7 +139,7 @@ func TestImportTestdata(t *testing.T) { tmpdir := mktmpdir(t) defer os.RemoveAll(tmpdir) - compile(t, "testdata", testfile, filepath.Join(tmpdir, "testdata")) + compile(t, "testdata", testfile, filepath.Join(tmpdir, "testdata"), nil) // filename should end with ".go" filename := testfile[:len(testfile)-3] @@ -156,6 +166,129 @@ func TestImportTestdata(t *testing.T) { } } +func TestImportTypeparamTests(t *testing.T) { + testenv.NeedsGo1Point(t, 18) // requires generics + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + } + + tmpdir := mktmpdir(t) + defer os.RemoveAll(tmpdir) + + // Check go files in test/typeparam, except those that fail for a known + // reason. + rootDir := filepath.Join(runtime.GOROOT(), "test", "typeparam") + list, err := os.ReadDir(rootDir) + if err != nil { + t.Fatal(err) + } + + var skip map[string]string + if !unifiedIR { + // The Go 1.18 frontend still fails several cases. + skip = map[string]string{ + "equal.go": "inconsistent embedded sorting", // TODO(rfindley): investigate this. + "nested.go": "fails to compile", // TODO(rfindley): investigate this. + "issue47631.go": "can not handle local type declarations", + "issue55101.go": "fails to compile", + } + } + + for _, entry := range list { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".go") { + // For now, only consider standalone go files. + continue + } + + t.Run(entry.Name(), func(t *testing.T) { + if reason, ok := skip[entry.Name()]; ok { + t.Skip(reason) + } + + filename := filepath.Join(rootDir, entry.Name()) + src, err := os.ReadFile(filename) + if err != nil { + t.Fatal(err) + } + if !bytes.HasPrefix(src, []byte("// run")) && !bytes.HasPrefix(src, []byte("// compile")) { + // We're bypassing the logic of run.go here, so be conservative about + // the files we consider in an attempt to make this test more robust to + // changes in test/typeparams. + t.Skipf("not detected as a run test") + } + + // Compile and import, and compare the resulting package with the package + // that was type-checked directly. + compile(t, rootDir, entry.Name(), filepath.Join(tmpdir, "testdata"), nil) + pkgName := strings.TrimSuffix(entry.Name(), ".go") + imported := importPkg(t, "./testdata/"+pkgName, tmpdir) + checked := checkFile(t, filename, src) + + seen := make(map[string]bool) + for _, name := range imported.Scope().Names() { + if !token.IsExported(name) { + continue // ignore synthetic names like .inittask and .dict.* + } + seen[name] = true + + importedObj := imported.Scope().Lookup(name) + got := types.ObjectString(importedObj, types.RelativeTo(imported)) + got = sanitizeObjectString(got) + + checkedObj := checked.Scope().Lookup(name) + if checkedObj == nil { + t.Fatalf("imported object %q was not type-checked", name) + } + want := types.ObjectString(checkedObj, types.RelativeTo(checked)) + want = sanitizeObjectString(want) + + if got != want { + t.Errorf("imported %q as %q, want %q", name, got, want) + } + } + + for _, name := range checked.Scope().Names() { + if !token.IsExported(name) || seen[name] { + continue + } + t.Errorf("did not import object %q", name) + } + }) + } +} + +// sanitizeObjectString removes type parameter debugging markers from an object +// string, to normalize it for comparison. +// TODO(rfindley): this should not be necessary. +func sanitizeObjectString(s string) string { + var runes []rune + for _, r := range s { + if '₀' <= r && r < '₀'+10 { + continue // trim type parameter subscripts + } + runes = append(runes, r) + } + return string(runes) +} + +func checkFile(t *testing.T, filename string, src []byte) *types.Package { + fset := token.NewFileSet() + f, err := goparser.ParseFile(fset, filename, src, 0) + if err != nil { + t.Fatal(err) + } + config := types.Config{ + Importer: goimporter.Default(), + } + pkg, err := config.Check("", fset, []*ast.File{f}, nil) + if err != nil { + t.Fatal(err) + } + return pkg +} + func TestVersionHandling(t *testing.T) { if debug { t.Skip("TestVersionHandling panics in debug mode") @@ -459,8 +592,8 @@ func TestIssue13566(t *testing.T) { if err != nil { t.Fatal(err) } - compilePkg(t, "testdata", "a.go", testoutdir, apkg(testoutdir)) - compile(t, testoutdir, bpath, testoutdir) + compilePkg(t, "testdata", "a.go", testoutdir, nil, apkg(testoutdir)) + compile(t, testoutdir, bpath, testoutdir, map[string]string{apkg(testoutdir): filepath.Join(testoutdir, "a.o")}) // import must succeed (test for issue at hand) pkg := importPkg(t, "./testdata/b", tmpdir) @@ -528,7 +661,7 @@ func TestIssue15517(t *testing.T) { tmpdir := mktmpdir(t) defer os.RemoveAll(tmpdir) - compile(t, "testdata", "p.go", filepath.Join(tmpdir, "testdata")) + compile(t, "testdata", "p.go", filepath.Join(tmpdir, "testdata"), nil) // Multiple imports of p must succeed without redeclaration errors. // We use an import path that's not cleaned up so that the eventual @@ -619,8 +752,8 @@ func TestIssue51836(t *testing.T) { if err != nil { t.Fatal(err) } - compilePkg(t, dir, "a.go", testoutdir, apkg(testoutdir)) - compile(t, testoutdir, bpath, testoutdir) + compilePkg(t, dir, "a.go", testoutdir, nil, apkg(testoutdir)) + compile(t, testoutdir, bpath, testoutdir, map[string]string{apkg(testoutdir): filepath.Join(testoutdir, "a.o")}) // import must succeed (test for issue at hand) _ = importPkg(t, "./testdata/aa", tmpdir) @@ -646,7 +779,7 @@ func importPkg(t *testing.T, path, srcDir string) *types.Package { func compileAndImportPkg(t *testing.T, name string) *types.Package { tmpdir := mktmpdir(t) defer os.RemoveAll(tmpdir) - compile(t, "testdata", name+".go", filepath.Join(tmpdir, "testdata")) + compile(t, "testdata", name+".go", filepath.Join(tmpdir, "testdata"), nil) return importPkg(t, "./testdata/"+name, tmpdir) } diff --git a/go/internal/gcimporter/iexport.go b/internal/gcimporter/iexport.go similarity index 90% rename from go/internal/gcimporter/iexport.go rename to internal/gcimporter/iexport.go index 9a4ff329e12..7d90f00f323 100644 --- a/go/internal/gcimporter/iexport.go +++ b/internal/gcimporter/iexport.go @@ -12,7 +12,6 @@ import ( "bytes" "encoding/binary" "fmt" - "go/ast" "go/constant" "go/token" "go/types" @@ -26,6 +25,41 @@ import ( "golang.org/x/tools/internal/typeparams" ) +// IExportShallow encodes "shallow" export data for the specified package. +// +// No promises are made about the encoding other than that it can be +// decoded by the same version of IIExportShallow. If you plan to save +// export data in the file system, be sure to include a cryptographic +// digest of the executable in the key to avoid version skew. +func IExportShallow(fset *token.FileSet, pkg *types.Package) ([]byte, error) { + // In principle this operation can only fail if out.Write fails, + // but that's impossible for bytes.Buffer---and as a matter of + // fact iexportCommon doesn't even check for I/O errors. + // TODO(adonovan): handle I/O errors properly. + // TODO(adonovan): use byte slices throughout, avoiding copying. + const bundle, shallow = false, true + var out bytes.Buffer + err := iexportCommon(&out, fset, bundle, shallow, iexportVersion, []*types.Package{pkg}) + return out.Bytes(), err +} + +// IImportShallow decodes "shallow" types.Package data encoded by IExportShallow +// in the same executable. This function cannot import data from +// cmd/compile or gcexportdata.Write. +func IImportShallow(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string, insert InsertType) (*types.Package, error) { + const bundle = false + pkgs, err := iimportCommon(fset, imports, data, bundle, path, insert) + if err != nil { + return nil, err + } + return pkgs[0], nil +} + +// InsertType is the type of a function that creates a types.TypeName +// object for a named type and inserts it into the scope of the +// specified Package. +type InsertType = func(pkg *types.Package, name string) + // Current bundled export format version. Increase with each format change. // 0: initial implementation const bundleVersion = 0 @@ -36,15 +70,17 @@ const bundleVersion = 0 // The package path of the top-level package will not be recorded, // so that calls to IImportData can override with a provided package path. func IExportData(out io.Writer, fset *token.FileSet, pkg *types.Package) error { - return iexportCommon(out, fset, false, iexportVersion, []*types.Package{pkg}) + const bundle, shallow = false, false + return iexportCommon(out, fset, bundle, shallow, iexportVersion, []*types.Package{pkg}) } // IExportBundle writes an indexed export bundle for pkgs to out. func IExportBundle(out io.Writer, fset *token.FileSet, pkgs []*types.Package) error { - return iexportCommon(out, fset, true, iexportVersion, pkgs) + const bundle, shallow = true, false + return iexportCommon(out, fset, bundle, shallow, iexportVersion, pkgs) } -func iexportCommon(out io.Writer, fset *token.FileSet, bundle bool, version int, pkgs []*types.Package) (err error) { +func iexportCommon(out io.Writer, fset *token.FileSet, bundle, shallow bool, version int, pkgs []*types.Package) (err error) { if !debug { defer func() { if e := recover(); e != nil { @@ -61,6 +97,7 @@ func iexportCommon(out io.Writer, fset *token.FileSet, bundle bool, version int, p := iexporter{ fset: fset, version: version, + shallow: shallow, allPkgs: map[*types.Package]bool{}, stringIndex: map[string]uint64{}, declIndex: map[types.Object]uint64{}, @@ -82,7 +119,7 @@ func iexportCommon(out io.Writer, fset *token.FileSet, bundle bool, version int, for _, pkg := range pkgs { scope := pkg.Scope() for _, name := range scope.Names() { - if ast.IsExported(name) { + if token.IsExported(name) { p.pushDecl(scope.Lookup(name)) } } @@ -205,7 +242,8 @@ type iexporter struct { out *bytes.Buffer version int - localpkg *types.Package + shallow bool // don't put types from other packages in the index + localpkg *types.Package // (nil in bundle mode) // allPkgs tracks all packages that have been referenced by // the export data, so we can ensure to include them in the @@ -256,6 +294,11 @@ func (p *iexporter) pushDecl(obj types.Object) { panic("cannot export package unsafe") } + // Shallow export data: don't index decls from other packages. + if p.shallow && obj.Pkg() != p.localpkg { + return + } + if _, ok := p.declIndex[obj]; ok { return } @@ -497,7 +540,7 @@ func (w *exportWriter) pkg(pkg *types.Package) { w.string(w.exportPath(pkg)) } -func (w *exportWriter) qualifiedIdent(obj types.Object) { +func (w *exportWriter) qualifiedType(obj *types.TypeName) { name := w.p.exportName(obj) // Ensure any referenced declarations are written out too. @@ -556,11 +599,11 @@ func (w *exportWriter) doTyp(t types.Type, pkg *types.Package) { return } w.startType(definedType) - w.qualifiedIdent(t.Obj()) + w.qualifiedType(t.Obj()) case *typeparams.TypeParam: w.startType(typeParamType) - w.qualifiedIdent(t.Obj()) + w.qualifiedType(t.Obj()) case *types.Pointer: w.startType(pointerType) @@ -602,14 +645,17 @@ func (w *exportWriter) doTyp(t types.Type, pkg *types.Package) { case *types.Struct: w.startType(structType) - w.setPkg(pkg, true) - n := t.NumFields() + if n > 0 { + w.setPkg(t.Field(0).Pkg(), true) // qualifying package for field objects + } else { + w.setPkg(pkg, true) + } w.uint64(uint64(n)) for i := 0; i < n; i++ { f := t.Field(i) w.pos(f.Pos()) - w.string(f.Name()) + w.string(f.Name()) // unexported fields implicitly qualified by prior setPkg w.typ(f.Type(), pkg) w.bool(f.Anonymous()) w.string(t.Tag(i)) // note (or tag) diff --git a/go/internal/gcimporter/iexport_common_test.go b/internal/gcimporter/iexport_common_test.go similarity index 100% rename from go/internal/gcimporter/iexport_common_test.go rename to internal/gcimporter/iexport_common_test.go diff --git a/go/internal/gcimporter/iexport_go118_test.go b/internal/gcimporter/iexport_go118_test.go similarity index 99% rename from go/internal/gcimporter/iexport_go118_test.go rename to internal/gcimporter/iexport_go118_test.go index 5dfa2580f6b..27ba8cec5ac 100644 --- a/go/internal/gcimporter/iexport_go118_test.go +++ b/internal/gcimporter/iexport_go118_test.go @@ -21,7 +21,7 @@ import ( "strings" "testing" - "golang.org/x/tools/go/internal/gcimporter" + "golang.org/x/tools/internal/gcimporter" ) // TODO(rfindley): migrate this to testdata, as has been done in the standard library. diff --git a/go/internal/gcimporter/iexport_test.go b/internal/gcimporter/iexport_test.go similarity index 85% rename from go/internal/gcimporter/iexport_test.go rename to internal/gcimporter/iexport_test.go index f0e83e519fe..93183f9dc6f 100644 --- a/go/internal/gcimporter/iexport_test.go +++ b/internal/gcimporter/iexport_test.go @@ -30,8 +30,9 @@ import ( "golang.org/x/tools/go/ast/inspector" "golang.org/x/tools/go/buildutil" - "golang.org/x/tools/go/internal/gcimporter" + "golang.org/x/tools/go/gcexportdata" "golang.org/x/tools/go/loader" + "golang.org/x/tools/internal/gcimporter" "golang.org/x/tools/internal/typeparams/genericfeatures" ) @@ -58,7 +59,8 @@ func readExportFile(filename string) ([]byte, error) { func iexport(fset *token.FileSet, version int, pkg *types.Package) ([]byte, error) { var buf bytes.Buffer - if err := gcimporter.IExportCommon(&buf, fset, false, version, []*types.Package{pkg}); err != nil { + const bundle, shallow = false, false + if err := gcimporter.IExportCommon(&buf, fset, bundle, shallow, version, []*types.Package{pkg}); err != nil { return nil, err } return buf.Bytes(), nil @@ -196,7 +198,7 @@ func testPkg(t *testing.T, fset *token.FileSet, version int, pkg *types.Package, // Compare the packages' corresponding members. for _, name := range pkg.Scope().Names() { - if !ast.IsExported(name) { + if !token.IsExported(name) { continue } obj1 := pkg.Scope().Lookup(name) @@ -403,3 +405,50 @@ func valueToRat(x constant.Value) *big.Rat { } return new(big.Rat).SetInt(new(big.Int).SetBytes(bytes)) } + +// This is a regression test for a bug in iexport of types.Struct: +// unexported fields were losing their implicit package qualifier. +func TestUnexportedStructFields(t *testing.T) { + fset := token.NewFileSet() + export := make(map[string][]byte) + + // process parses and type-checks a single-file + // package and saves its export data. + process := func(path, content string) { + syntax, err := parser.ParseFile(fset, path+"/x.go", content, 0) + if err != nil { + t.Fatal(err) + } + packages := make(map[string]*types.Package) // keys are package paths + cfg := &types.Config{ + Importer: importerFunc(func(path string) (*types.Package, error) { + data, ok := export[path] + if !ok { + return nil, fmt.Errorf("missing export data for %s", path) + } + return gcexportdata.Read(bytes.NewReader(data), fset, packages, path) + }), + } + pkg := types.NewPackage(path, syntax.Name.Name) + check := types.NewChecker(cfg, fset, pkg, nil) + if err := check.Files([]*ast.File{syntax}); err != nil { + t.Fatal(err) + } + var out bytes.Buffer + if err := gcexportdata.Write(&out, fset, pkg); err != nil { + t.Fatal(err) + } + export[path] = out.Bytes() + } + + // Historically this led to a spurious error: + // "cannot convert a.M (variable of type a.MyTime) to type time.Time" + // because the private fields of Time and MyTime were not identical. + process("time", `package time; type Time struct { x, y int }`) + process("a", `package a; import "time"; type MyTime time.Time; var M MyTime`) + process("b", `package b; import ("a"; "time"); var _ = time.Time(a.M)`) +} + +type importerFunc func(path string) (*types.Package, error) + +func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) } diff --git a/go/internal/gcimporter/iimport.go b/internal/gcimporter/iimport.go similarity index 96% rename from go/internal/gcimporter/iimport.go rename to internal/gcimporter/iimport.go index 6e4c066b69b..a1c46965350 100644 --- a/go/internal/gcimporter/iimport.go +++ b/internal/gcimporter/iimport.go @@ -85,7 +85,7 @@ const ( // If the export data version is not recognized or the format is otherwise // compromised, an error is returned. func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (int, *types.Package, error) { - pkgs, err := iimportCommon(fset, imports, data, false, path) + pkgs, err := iimportCommon(fset, imports, data, false, path, nil) if err != nil { return 0, nil, err } @@ -94,10 +94,10 @@ func IImportData(fset *token.FileSet, imports map[string]*types.Package, data [] // IImportBundle imports a set of packages from the serialized package bundle. func IImportBundle(fset *token.FileSet, imports map[string]*types.Package, data []byte) ([]*types.Package, error) { - return iimportCommon(fset, imports, data, true, "") + return iimportCommon(fset, imports, data, true, "", nil) } -func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data []byte, bundle bool, path string) (pkgs []*types.Package, err error) { +func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data []byte, bundle bool, path string, insert InsertType) (pkgs []*types.Package, err error) { const currentVersion = iexportVersionCurrent version := int64(-1) if !debug { @@ -147,6 +147,7 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data p := iimporter{ version: int(version), ipath: path, + insert: insert, stringData: stringData, stringCache: make(map[uint64]string), @@ -187,11 +188,18 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data } else if pkg.Name() != pkgName { errorf("conflicting names %s and %s for package %q", pkg.Name(), pkgName, path) } + if i == 0 && !bundle { + p.localpkg = pkg + } p.pkgCache[pkgPathOff] = pkg + // Read index for package. nameIndex := make(map[string]uint64) - for nSyms := r.uint64(); nSyms > 0; nSyms-- { + nSyms := r.uint64() + // In shallow mode we don't expect an index for other packages. + assert(nSyms == 0 || p.localpkg == pkg || p.insert == nil) + for ; nSyms > 0; nSyms-- { name := p.stringAt(r.uint64()) nameIndex[name] = r.uint64() } @@ -267,6 +275,9 @@ type iimporter struct { version int ipath string + localpkg *types.Package + insert func(pkg *types.Package, name string) // "shallow" mode only + stringData []byte stringCache map[uint64]string pkgCache map[uint64]*types.Package @@ -310,6 +321,13 @@ func (p *iimporter) doDecl(pkg *types.Package, name string) { off, ok := p.pkgIndex[pkg][name] if !ok { + // In "shallow" mode, call back to the application to + // find the object and insert it into the package scope. + if p.insert != nil { + assert(pkg != p.localpkg) + p.insert(pkg, name) // "can't fail" + return + } errorf("%v.%v not in index", pkg, name) } diff --git a/go/internal/gcimporter/israce_test.go b/internal/gcimporter/israce_test.go similarity index 100% rename from go/internal/gcimporter/israce_test.go rename to internal/gcimporter/israce_test.go diff --git a/go/internal/gcimporter/newInterface10.go b/internal/gcimporter/newInterface10.go similarity index 100% rename from go/internal/gcimporter/newInterface10.go rename to internal/gcimporter/newInterface10.go diff --git a/go/internal/gcimporter/newInterface11.go b/internal/gcimporter/newInterface11.go similarity index 100% rename from go/internal/gcimporter/newInterface11.go rename to internal/gcimporter/newInterface11.go diff --git a/internal/gcimporter/shallow_test.go b/internal/gcimporter/shallow_test.go new file mode 100644 index 00000000000..3d8c86a1545 --- /dev/null +++ b/internal/gcimporter/shallow_test.go @@ -0,0 +1,163 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gcimporter_test + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "testing" + + "golang.org/x/sync/errgroup" + "golang.org/x/tools/go/packages" + "golang.org/x/tools/internal/gcimporter" + "golang.org/x/tools/internal/testenv" +) + +// TestStd type-checks the standard library using shallow export data. +func TestShallowStd(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode; too slow (https://golang.org/issue/14113)") + } + testenv.NeedsTool(t, "go") + + // Load import graph of the standard library. + // (No parsing or type-checking.) + cfg := &packages.Config{ + Mode: packages.NeedImports | + packages.NeedName | + packages.NeedFiles | // see https://github.com/golang/go/issues/56632 + packages.NeedCompiledGoFiles, + Tests: false, + } + pkgs, err := packages.Load(cfg, "std") + if err != nil { + t.Fatalf("load: %v", err) + } + if len(pkgs) < 200 { + t.Fatalf("too few packages: %d", len(pkgs)) + } + + // Type check the packages in parallel postorder. + done := make(map[*packages.Package]chan struct{}) + packages.Visit(pkgs, nil, func(p *packages.Package) { + done[p] = make(chan struct{}) + }) + packages.Visit(pkgs, nil, + func(pkg *packages.Package) { + go func() { + // Wait for all deps to be done. + for _, imp := range pkg.Imports { + <-done[imp] + } + typecheck(t, pkg) + close(done[pkg]) + }() + }) + for _, root := range pkgs { + <-done[root] + } +} + +// typecheck reads, parses, and type-checks a package. +// It squirrels the export data in the the ppkg.ExportFile field. +func typecheck(t *testing.T, ppkg *packages.Package) { + if ppkg.PkgPath == "unsafe" { + return // unsafe is special + } + + // Create a local FileSet just for this package. + fset := token.NewFileSet() + + // Parse files in parallel. + syntax := make([]*ast.File, len(ppkg.CompiledGoFiles)) + var group errgroup.Group + for i, filename := range ppkg.CompiledGoFiles { + i, filename := i, filename + group.Go(func() error { + f, err := parser.ParseFile(fset, filename, nil, parser.SkipObjectResolution) + if err != nil { + return err // e.g. missing file + } + syntax[i] = f + return nil + }) + } + if err := group.Wait(); err != nil { + t.Fatal(err) + } + // Inv: all files were successfully parsed. + + // Build map of dependencies by package path. + // (We don't compute this mapping for the entire + // packages graph because it is not globally consistent.) + depsByPkgPath := make(map[string]*packages.Package) + { + var visit func(*packages.Package) + visit = func(pkg *packages.Package) { + if depsByPkgPath[pkg.PkgPath] == nil { + depsByPkgPath[pkg.PkgPath] = pkg + for path := range pkg.Imports { + visit(pkg.Imports[path]) + } + } + } + visit(ppkg) + } + + // importer state + var ( + insert func(p *types.Package, name string) + importMap = make(map[string]*types.Package) // keys are PackagePaths + ) + loadFromExportData := func(imp *packages.Package) (*types.Package, error) { + data := []byte(imp.ExportFile) + return gcimporter.IImportShallow(fset, importMap, data, imp.PkgPath, insert) + } + insert = func(p *types.Package, name string) { + imp, ok := depsByPkgPath[p.Path()] + if !ok { + t.Fatalf("can't find dependency: %q", p.Path()) + } + imported, err := loadFromExportData(imp) + if err != nil { + t.Fatalf("unmarshal: %v", err) + } + if imported != p { + t.Fatalf("internal error: inconsistent packages") + } + if obj := imported.Scope().Lookup(name); obj == nil { + t.Fatalf("lookup %q.%s failed", imported.Path(), name) + } + } + + cfg := &types.Config{ + Error: func(e error) { + t.Error(e) + }, + Importer: importerFunc(func(importPath string) (*types.Package, error) { + if importPath == "unsafe" { + return types.Unsafe, nil // unsafe has no exportdata + } + imp, ok := ppkg.Imports[importPath] + if !ok { + return nil, fmt.Errorf("missing import %q", importPath) + } + return loadFromExportData(imp) + }), + } + + // Type-check the syntax trees. + tpkg, _ := cfg.Check(ppkg.PkgPath, fset, syntax, nil) + + // Save the export data. + data, err := gcimporter.IExportShallow(fset, tpkg) + if err != nil { + t.Fatalf("internal error marshalling export data: %v", err) + } + ppkg.ExportFile = string(data) +} diff --git a/internal/gcimporter/stdlib_test.go b/internal/gcimporter/stdlib_test.go new file mode 100644 index 00000000000..ec1be3ea031 --- /dev/null +++ b/internal/gcimporter/stdlib_test.go @@ -0,0 +1,82 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gcimporter_test + +import ( + "bytes" + "fmt" + "go/token" + "go/types" + "testing" + "unsafe" + + "golang.org/x/tools/go/gcexportdata" + "golang.org/x/tools/go/packages" + "golang.org/x/tools/internal/testenv" +) + +// TestStdlib ensures that all packages in std and x/tools can be +// type-checked using export data. Takes around 3s. +func TestStdlib(t *testing.T) { + testenv.NeedsGoPackages(t) + + // gcexportdata.Read rapidly consumes FileSet address space, + // so disable the test on 32-bit machines. + // (We could use a fresh FileSet per type-check, but that + // would require us to re-parse the source using it.) + if unsafe.Sizeof(token.NoPos) < 8 { + t.Skip("skipping test on 32-bit machine") + } + + // Load, parse and type-check the standard library and x/tools. + cfg := &packages.Config{Mode: packages.LoadAllSyntax} + pkgs, err := packages.Load(cfg, "std", "golang.org/x/tools/...") + if err != nil { + t.Fatalf("failed to load/parse/type-check: %v", err) + } + if packages.PrintErrors(pkgs) > 0 { + t.Fatal("there were errors during loading") + } + if len(pkgs) < 240 { + t.Errorf("too few packages (%d) were loaded", len(pkgs)) + } + + export := make(map[string][]byte) // keys are package IDs + + // Re-type check them all in post-order, using export data. + packages.Visit(pkgs, nil, func(pkg *packages.Package) { + packages := make(map[string]*types.Package) // keys are package paths + cfg := &types.Config{ + Error: func(e error) { + t.Errorf("type error: %v", e) + }, + Importer: importerFunc(func(importPath string) (*types.Package, error) { + // Resolve import path to (vendored?) package path. + imported := pkg.Imports[importPath] + + if imported.PkgPath == "unsafe" { + return types.Unsafe, nil // unsafe has no exportdata + } + + data, ok := export[imported.ID] + if !ok { + return nil, fmt.Errorf("missing export data for %s", importPath) + } + return gcexportdata.Read(bytes.NewReader(data), pkg.Fset, packages, imported.PkgPath) + }), + } + + // Re-typecheck the syntax and save the export data in the map. + newPkg := types.NewPackage(pkg.PkgPath, pkg.Name) + check := types.NewChecker(cfg, pkg.Fset, newPkg, nil) + check.Files(pkg.Syntax) + + var out bytes.Buffer + if err := gcexportdata.Write(&out, pkg.Fset, newPkg); err != nil { + t.Fatalf("internal error writing export data: %v", err) + } + export[pkg.ID] = out.Bytes() + }) +} diff --git a/go/internal/gcimporter/support_go117.go b/internal/gcimporter/support_go117.go similarity index 100% rename from go/internal/gcimporter/support_go117.go rename to internal/gcimporter/support_go117.go diff --git a/go/internal/gcimporter/support_go118.go b/internal/gcimporter/support_go118.go similarity index 62% rename from go/internal/gcimporter/support_go118.go rename to internal/gcimporter/support_go118.go index a993843230c..edbe6ea7041 100644 --- a/go/internal/gcimporter/support_go118.go +++ b/internal/gcimporter/support_go118.go @@ -21,3 +21,17 @@ func additionalPredeclared() []types.Type { types.Universe.Lookup("any").Type(), } } + +// See cmd/compile/internal/types.SplitVargenSuffix. +func splitVargenSuffix(name string) (base, suffix string) { + i := len(name) + for i > 0 && name[i-1] >= '0' && name[i-1] <= '9' { + i-- + } + const dot = "·" + if i >= len(dot) && name[i-len(dot):i] == dot { + i -= len(dot) + return name[:i], name[i:] + } + return name, "" +} diff --git a/go/internal/gcimporter/testdata/a.go b/internal/gcimporter/testdata/a.go similarity index 100% rename from go/internal/gcimporter/testdata/a.go rename to internal/gcimporter/testdata/a.go diff --git a/go/internal/gcimporter/testdata/b.go b/internal/gcimporter/testdata/b.go similarity index 100% rename from go/internal/gcimporter/testdata/b.go rename to internal/gcimporter/testdata/b.go diff --git a/go/internal/gcimporter/testdata/exports.go b/internal/gcimporter/testdata/exports.go similarity index 100% rename from go/internal/gcimporter/testdata/exports.go rename to internal/gcimporter/testdata/exports.go diff --git a/go/internal/gcimporter/testdata/issue15920.go b/internal/gcimporter/testdata/issue15920.go similarity index 100% rename from go/internal/gcimporter/testdata/issue15920.go rename to internal/gcimporter/testdata/issue15920.go diff --git a/go/internal/gcimporter/testdata/issue20046.go b/internal/gcimporter/testdata/issue20046.go similarity index 100% rename from go/internal/gcimporter/testdata/issue20046.go rename to internal/gcimporter/testdata/issue20046.go diff --git a/go/internal/gcimporter/testdata/issue25301.go b/internal/gcimporter/testdata/issue25301.go similarity index 100% rename from go/internal/gcimporter/testdata/issue25301.go rename to internal/gcimporter/testdata/issue25301.go diff --git a/go/internal/gcimporter/testdata/issue51836/a.go b/internal/gcimporter/testdata/issue51836/a.go similarity index 100% rename from go/internal/gcimporter/testdata/issue51836/a.go rename to internal/gcimporter/testdata/issue51836/a.go diff --git a/go/internal/gcimporter/testdata/issue51836/aa.go b/internal/gcimporter/testdata/issue51836/aa.go similarity index 100% rename from go/internal/gcimporter/testdata/issue51836/aa.go rename to internal/gcimporter/testdata/issue51836/aa.go diff --git a/go/internal/gcimporter/testdata/p.go b/internal/gcimporter/testdata/p.go similarity index 100% rename from go/internal/gcimporter/testdata/p.go rename to internal/gcimporter/testdata/p.go diff --git a/go/internal/gcimporter/testdata/versions/test.go b/internal/gcimporter/testdata/versions/test.go similarity index 100% rename from go/internal/gcimporter/testdata/versions/test.go rename to internal/gcimporter/testdata/versions/test.go diff --git a/go/internal/gcimporter/testdata/versions/test_go1.11_0i.a b/internal/gcimporter/testdata/versions/test_go1.11_0i.a similarity index 100% rename from go/internal/gcimporter/testdata/versions/test_go1.11_0i.a rename to internal/gcimporter/testdata/versions/test_go1.11_0i.a diff --git a/go/internal/gcimporter/testdata/versions/test_go1.11_6b.a b/internal/gcimporter/testdata/versions/test_go1.11_6b.a similarity index 100% rename from go/internal/gcimporter/testdata/versions/test_go1.11_6b.a rename to internal/gcimporter/testdata/versions/test_go1.11_6b.a diff --git a/go/internal/gcimporter/testdata/versions/test_go1.11_999b.a b/internal/gcimporter/testdata/versions/test_go1.11_999b.a similarity index 100% rename from go/internal/gcimporter/testdata/versions/test_go1.11_999b.a rename to internal/gcimporter/testdata/versions/test_go1.11_999b.a diff --git a/go/internal/gcimporter/testdata/versions/test_go1.11_999i.a b/internal/gcimporter/testdata/versions/test_go1.11_999i.a similarity index 100% rename from go/internal/gcimporter/testdata/versions/test_go1.11_999i.a rename to internal/gcimporter/testdata/versions/test_go1.11_999i.a diff --git a/go/internal/gcimporter/testdata/versions/test_go1.7_0.a b/internal/gcimporter/testdata/versions/test_go1.7_0.a similarity index 100% rename from go/internal/gcimporter/testdata/versions/test_go1.7_0.a rename to internal/gcimporter/testdata/versions/test_go1.7_0.a diff --git a/go/internal/gcimporter/testdata/versions/test_go1.7_1.a b/internal/gcimporter/testdata/versions/test_go1.7_1.a similarity index 100% rename from go/internal/gcimporter/testdata/versions/test_go1.7_1.a rename to internal/gcimporter/testdata/versions/test_go1.7_1.a diff --git a/go/internal/gcimporter/testdata/versions/test_go1.8_4.a b/internal/gcimporter/testdata/versions/test_go1.8_4.a similarity index 100% rename from go/internal/gcimporter/testdata/versions/test_go1.8_4.a rename to internal/gcimporter/testdata/versions/test_go1.8_4.a diff --git a/go/internal/gcimporter/testdata/versions/test_go1.8_5.a b/internal/gcimporter/testdata/versions/test_go1.8_5.a similarity index 100% rename from go/internal/gcimporter/testdata/versions/test_go1.8_5.a rename to internal/gcimporter/testdata/versions/test_go1.8_5.a diff --git a/go/internal/gcimporter/unified_no.go b/internal/gcimporter/unified_no.go similarity index 100% rename from go/internal/gcimporter/unified_no.go rename to internal/gcimporter/unified_no.go diff --git a/go/internal/gcimporter/unified_yes.go b/internal/gcimporter/unified_yes.go similarity index 100% rename from go/internal/gcimporter/unified_yes.go rename to internal/gcimporter/unified_yes.go diff --git a/go/internal/gcimporter/ureader_no.go b/internal/gcimporter/ureader_no.go similarity index 100% rename from go/internal/gcimporter/ureader_no.go rename to internal/gcimporter/ureader_no.go diff --git a/go/internal/gcimporter/ureader_yes.go b/internal/gcimporter/ureader_yes.go similarity index 98% rename from go/internal/gcimporter/ureader_yes.go rename to internal/gcimporter/ureader_yes.go index 2d421c9619d..e09053bd37a 100644 --- a/go/internal/gcimporter/ureader_yes.go +++ b/internal/gcimporter/ureader_yes.go @@ -14,7 +14,7 @@ import ( "go/types" "strings" - "golang.org/x/tools/go/internal/pkgbits" + "golang.org/x/tools/internal/pkgbits" ) // A pkgReader holds the shared state for reading a unified IR package @@ -490,6 +490,11 @@ func (pr *pkgReader) objIdx(idx pkgbits.Index) (*types.Package, string) { return objPkg, objName } + // Ignore local types promoted to global scope (#55110). + if _, suffix := splitVargenSuffix(objName); suffix != "" { + return objPkg, objName + } + if objPkg.Scope().Lookup(objName) == nil { dict := pr.objDictIdx(idx) diff --git a/internal/goroot/importcfg.go b/internal/goroot/importcfg.go new file mode 100644 index 00000000000..6575cfb9df6 --- /dev/null +++ b/internal/goroot/importcfg.go @@ -0,0 +1,71 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package goroot is a copy of package internal/goroot +// in the main GO repot. It provides a utility to produce +// an importcfg and import path to package file map mapping +// standard library packages to the locations of their export +// data files. +package goroot + +import ( + "bytes" + "fmt" + "os/exec" + "strings" + "sync" +) + +// Importcfg returns an importcfg file to be passed to the +// Go compiler that contains the cached paths for the .a files for the +// standard library. +func Importcfg() (string, error) { + var icfg bytes.Buffer + + m, err := PkgfileMap() + if err != nil { + return "", err + } + fmt.Fprintf(&icfg, "# import config") + for importPath, export := range m { + if importPath != "unsafe" && export != "" { // unsafe + fmt.Fprintf(&icfg, "\npackagefile %s=%s", importPath, export) + } + } + s := icfg.String() + return s, nil +} + +var ( + stdlibPkgfileMap map[string]string + stdlibPkgfileErr error + once sync.Once +) + +// PkgfileMap returns a map of package paths to the location on disk +// of the .a file for the package. +// The caller must not modify the map. +func PkgfileMap() (map[string]string, error) { + once.Do(func() { + m := make(map[string]string) + output, err := exec.Command("go", "list", "-export", "-e", "-f", "{{.ImportPath}} {{.Export}}", "std", "cmd").Output() + if err != nil { + stdlibPkgfileErr = err + } + for _, line := range strings.Split(string(output), "\n") { + if line == "" { + continue + } + sp := strings.SplitN(line, " ", 2) + if len(sp) != 2 { + err = fmt.Errorf("determining pkgfile map: invalid line in go list output: %q", line) + return + } + importPath, export := sp[0], sp[1] + m[importPath] = export + } + stdlibPkgfileMap = m + }) + return stdlibPkgfileMap, stdlibPkgfileErr +} diff --git a/internal/jsonrpc2_v2/conn.go b/internal/jsonrpc2_v2/conn.go index 7995f404e58..60afa7060e4 100644 --- a/internal/jsonrpc2_v2/conn.go +++ b/internal/jsonrpc2_v2/conn.go @@ -7,12 +7,15 @@ package jsonrpc2 import ( "context" "encoding/json" + "errors" "fmt" "io" "sync" "sync/atomic" + "time" "golang.org/x/tools/internal/event" + "golang.org/x/tools/internal/event/keys" "golang.org/x/tools/internal/event/label" "golang.org/x/tools/internal/event/tag" ) @@ -24,14 +27,16 @@ import ( type Binder interface { // Bind returns the ConnectionOptions to use when establishing the passed-in // Connection. - // The connection is not ready to use when Bind is called. - Bind(context.Context, *Connection) (ConnectionOptions, error) + // + // The connection is not ready to use when Bind is called, + // but Bind may close it without reading or writing to it. + Bind(context.Context, *Connection) ConnectionOptions } // A BinderFunc implements the Binder interface for a standalone Bind function. -type BinderFunc func(context.Context, *Connection) (ConnectionOptions, error) +type BinderFunc func(context.Context, *Connection) ConnectionOptions -func (f BinderFunc) Bind(ctx context.Context, c *Connection) (ConnectionOptions, error) { +func (f BinderFunc) Bind(ctx context.Context, c *Connection) ConnectionOptions { return f(ctx, c) } @@ -48,6 +53,10 @@ type ConnectionOptions struct { // Handler is used as the queued message handler for inbound messages. // If nil, all responses will be ErrNotHandled. Handler Handler + // OnInternalError, if non-nil, is called with any internal errors that occur + // while serving the connection, such as protocol errors or invariant + // violations. (If nil, internal errors result in panics.) + OnInternalError func(error) } // Connection manages the jsonrpc2 protocol, connecting responses back to their @@ -57,103 +66,242 @@ type ConnectionOptions struct { type Connection struct { seq int64 // must only be accessed using atomic operations - closeOnce sync.Once - closer io.Closer + stateMu sync.Mutex + state inFlightState // accessed only in updateInFlight + done chan struct{} // closed (under stateMu) when state.closed is true and all goroutines have completed - writer chan Writer - outgoing chan map[ID]chan<- *Response - incoming chan map[ID]*incoming - async *async + writer chan Writer // 1-buffered; stores the writer when not in use + + handler Handler + + onInternalError func(error) + onDone func() } -type AsyncCall struct { - id ID - response chan *Response // the channel a response will be delivered on - result chan asyncResult - endSpan func() // close the tracing span when all processing for the message is complete +// inFlightState records the state of the incoming and outgoing calls on a +// Connection. +type inFlightState struct { + connClosing bool // true when the Connection's Close method has been called + reading bool // true while the readIncoming goroutine is running + readErr error // non-nil when the readIncoming goroutine exits (typically io.EOF) + writeErr error // non-nil if a call to the Writer has failed with a non-canceled Context + + // closer shuts down and cleans up the Reader and Writer state, ideally + // interrupting any Read or Write call that is currently blocked. It is closed + // when the state is idle and one of: connClosing is true, readErr is non-nil, + // or writeErr is non-nil. + // + // After the closer has been invoked, the closer field is set to nil + // and the closeErr field is simultaneously set to its result. + closer io.Closer + closeErr error // error returned from closer.Close + + outgoingCalls map[ID]*AsyncCall // calls only + outgoingNotifications int // # of notifications awaiting "write" + + // incoming stores the total number of incoming calls and notifications + // that have not yet written or processed a result. + incoming int + + incomingByID map[ID]*incomingRequest // calls only + + // handlerQueue stores the backlog of calls and notifications that were not + // already handled by a preempter. + // The queue does not include the request currently being handled (if any). + handlerQueue []*incomingRequest + handlerRunning bool +} + +// updateInFlight locks the state of the connection's in-flight requests, allows +// f to mutate that state, and closes the connection if it is idle and either +// is closing or has a read or write error. +func (c *Connection) updateInFlight(f func(*inFlightState)) { + c.stateMu.Lock() + defer c.stateMu.Unlock() + + s := &c.state + + f(s) + + select { + case <-c.done: + // The connection was already completely done at the start of this call to + // updateInFlight, so it must remain so. (The call to f should have noticed + // that and avoided making any updates that would cause the state to be + // non-idle.) + if !s.idle() { + panic("jsonrpc2_v2: updateInFlight transitioned to non-idle when already done") + } + return + default: + } + + if s.idle() && s.shuttingDown(ErrUnknown) != nil { + if s.closer != nil { + s.closeErr = s.closer.Close() + s.closer = nil // prevent duplicate Close calls + } + if s.reading { + // The readIncoming goroutine is still running. Our call to Close should + // cause it to exit soon, at which point it will make another call to + // updateInFlight, set s.reading to false, and mark the Connection done. + } else { + // The readIncoming goroutine has exited, or never started to begin with. + // Since everything else is idle, we're completely done. + if c.onDone != nil { + c.onDone() + } + close(c.done) + } + } } -type asyncResult struct { - result []byte - err error +// idle reports whether the connction is in a state with no pending calls or +// notifications. +// +// If idle returns true, the readIncoming goroutine may still be running, +// but no other goroutines are doing work on behalf of the connnection. +func (s *inFlightState) idle() bool { + return len(s.outgoingCalls) == 0 && s.outgoingNotifications == 0 && s.incoming == 0 && !s.handlerRunning } -// incoming is used to track an incoming request as it is being handled -type incoming struct { - request *Request // the request being processed - baseCtx context.Context // a base context for the message processing - done func() // a function called when all processing for the message is complete - handleCtx context.Context // the context for handling the message, child of baseCtx - cancel func() // a function that cancels the handling context +// shuttingDown reports whether the connection is in a state that should +// disallow new (incoming and outgoing) calls. It returns either nil or +// an error that is or wraps the provided errClosing. +func (s *inFlightState) shuttingDown(errClosing error) error { + if s.connClosing { + // If Close has been called explicitly, it doesn't matter what state the + // Reader and Writer are in: we shouldn't be starting new work because the + // caller told us not to start new work. + return errClosing + } + if s.readErr != nil { + // If the read side of the connection is broken, we cannot read new call + // requests, and cannot read responses to our outgoing calls. + return fmt.Errorf("%w: %v", errClosing, s.readErr) + } + if s.writeErr != nil { + // If the write side of the connection is broken, we cannot write responses + // for incoming calls, and cannot write requests for outgoing calls. + return fmt.Errorf("%w: %v", errClosing, s.writeErr) + } + return nil +} + +// incomingRequest is used to track an incoming request as it is being handled +type incomingRequest struct { + *Request // the request being processed + ctx context.Context + cancel context.CancelFunc + endSpan func() // called (and set to nil) when the response is sent } // Bind returns the options unmodified. -func (o ConnectionOptions) Bind(context.Context, *Connection) (ConnectionOptions, error) { - return o, nil +func (o ConnectionOptions) Bind(context.Context, *Connection) ConnectionOptions { + return o } // newConnection creates a new connection and runs it. +// // This is used by the Dial and Serve functions to build the actual connection. -func newConnection(ctx context.Context, rwc io.ReadWriteCloser, binder Binder) (*Connection, error) { +// +// The connection is closed automatically (and its resources cleaned up) when +// the last request has completed after the underlying ReadWriteCloser breaks, +// but it may be stopped earlier by calling Close (for a clean shutdown). +func newConnection(bindCtx context.Context, rwc io.ReadWriteCloser, binder Binder, onDone func()) *Connection { + // TODO: Should we create a new event span here? + // This will propagate cancellation from ctx; should it? + ctx := notDone{bindCtx} + c := &Connection{ - closer: rwc, - writer: make(chan Writer, 1), - outgoing: make(chan map[ID]chan<- *Response, 1), - incoming: make(chan map[ID]*incoming, 1), - async: newAsync(), + state: inFlightState{closer: rwc}, + done: make(chan struct{}), + writer: make(chan Writer, 1), + onDone: onDone, + } + // It's tempting to set a finalizer on c to verify that the state has gone + // idle when the connection becomes unreachable. Unfortunately, the Binder + // interface makes that unsafe: it allows the Handler to close over the + // Connection, which could create a reference cycle that would cause the + // Connection to become uncollectable. + + options := binder.Bind(bindCtx, c) + framer := options.Framer + if framer == nil { + framer = HeaderFramer() + } + c.handler = options.Handler + if c.handler == nil { + c.handler = defaultHandler{} } + c.onInternalError = options.OnInternalError - options, err := binder.Bind(ctx, c) - if err != nil { - return nil, err - } - if options.Framer == nil { - options.Framer = HeaderFramer() - } - if options.Preempter == nil { - options.Preempter = defaultHandler{} - } - if options.Handler == nil { - options.Handler = defaultHandler{} - } - c.outgoing <- make(map[ID]chan<- *Response) - c.incoming <- make(map[ID]*incoming) - // the goroutines started here will continue until the underlying stream is closed - reader := options.Framer.Reader(rwc) - readToQueue := make(chan *incoming) - queueToDeliver := make(chan *incoming) - go c.readIncoming(ctx, reader, readToQueue) - go c.manageQueue(ctx, options.Preempter, readToQueue, queueToDeliver) - go c.deliverMessages(ctx, options.Handler, queueToDeliver) - - // releaseing the writer must be the last thing we do in case any requests - // are blocked waiting for the connection to be ready - c.writer <- options.Framer.Writer(rwc) - return c, nil + c.writer <- framer.Writer(rwc) + reader := framer.Reader(rwc) + + c.updateInFlight(func(s *inFlightState) { + select { + case <-c.done: + // Bind already closed the connection; don't start a goroutine to read it. + return + default: + } + + // The goroutine started here will continue until the underlying stream is closed. + // + // (If the Binder closed the Connection already, this should error out and + // return almost immediately.) + s.reading = true + go c.readIncoming(ctx, reader, options.Preempter) + }) + return c } // Notify invokes the target method but does not wait for a response. // The params will be marshaled to JSON before sending over the wire, and will // be handed to the method invoked. -func (c *Connection) Notify(ctx context.Context, method string, params interface{}) error { - notify, err := NewNotification(method, params) - if err != nil { - return fmt.Errorf("marshaling notify parameters: %v", err) - } +func (c *Connection) Notify(ctx context.Context, method string, params interface{}) (err error) { ctx, done := event.Start(ctx, method, tag.Method.Of(method), tag.RPCDirection.Of(tag.Outbound), ) - event.Metric(ctx, tag.Started.Of(1)) - err = c.write(ctx, notify) - switch { - case err != nil: - event.Label(ctx, tag.StatusCode.Of("ERROR")) - default: - event.Label(ctx, tag.StatusCode.Of("OK")) + attempted := false + + defer func() { + labelStatus(ctx, err) + done() + if attempted { + c.updateInFlight(func(s *inFlightState) { + s.outgoingNotifications-- + }) + } + }() + + c.updateInFlight(func(s *inFlightState) { + // If the connection is shutting down, allow outgoing notifications only if + // there is at least one call still in flight. The number of calls in flight + // cannot increase once shutdown begins, and allowing outgoing notifications + // may permit notifications that will cancel in-flight calls. + if len(s.outgoingCalls) == 0 && len(s.incomingByID) == 0 { + err = s.shuttingDown(ErrClientClosing) + if err != nil { + return + } + } + s.outgoingNotifications++ + attempted = true + }) + if err != nil { + return err } - done() - return err + + notify, err := NewNotification(method, params) + if err != nil { + return fmt.Errorf("marshaling notify parameters: %v", err) + } + + event.Metric(ctx, tag.Started.Of(1)) + return c.write(ctx, notify) } // Call invokes the target method and returns an object that can be used to await the response. @@ -162,375 +310,442 @@ func (c *Connection) Notify(ctx context.Context, method string, params interface // You do not have to wait for the response, it can just be ignored if not needed. // If sending the call failed, the response will be ready and have the error in it. func (c *Connection) Call(ctx context.Context, method string, params interface{}) *AsyncCall { - result := &AsyncCall{ - id: Int64ID(atomic.AddInt64(&c.seq, 1)), - result: make(chan asyncResult, 1), - } - // generate a new request identifier - call, err := NewCall(result.id, method, params) - if err != nil { - //set the result to failed - result.result <- asyncResult{err: fmt.Errorf("marshaling call parameters: %w", err)} - return result - } + // Generate a new request identifier. + id := Int64ID(atomic.AddInt64(&c.seq, 1)) ctx, endSpan := event.Start(ctx, method, tag.Method.Of(method), tag.RPCDirection.Of(tag.Outbound), - tag.RPCID.Of(fmt.Sprintf("%q", result.id)), + tag.RPCID.Of(fmt.Sprintf("%q", id)), ) - result.endSpan = endSpan - event.Metric(ctx, tag.Started.Of(1)) - // We have to add ourselves to the pending map before we send, otherwise we - // are racing the response. - // rchan is buffered in case the response arrives without a listener. - result.response = make(chan *Response, 1) - outgoing, ok := <-c.outgoing - if !ok { - // If the call failed due to (say) an I/O error or broken pipe, attribute it - // as such. (If the error is nil, then the connection must have been shut - // down cleanly.) - err := c.async.wait() - if err == nil { - err = ErrClientClosing - } - resp, respErr := NewResponse(result.id, nil, err) - if respErr != nil { - panic(fmt.Errorf("unexpected error from NewResponse: %w", respErr)) + ac := &AsyncCall{ + id: id, + ready: make(chan struct{}), + ctx: ctx, + endSpan: endSpan, + } + // When this method returns, either ac is retired, or the request has been + // written successfully and the call is awaiting a response (to be provided by + // the readIncoming goroutine). + + call, err := NewCall(ac.id, method, params) + if err != nil { + ac.retire(&Response{ID: id, Error: fmt.Errorf("marshaling call parameters: %w", err)}) + return ac + } + + c.updateInFlight(func(s *inFlightState) { + err = s.shuttingDown(ErrClientClosing) + if err != nil { + return + } + if s.outgoingCalls == nil { + s.outgoingCalls = make(map[ID]*AsyncCall) } - result.response <- resp - return result + s.outgoingCalls[ac.id] = ac + }) + if err != nil { + ac.retire(&Response{ID: id, Error: err}) + return ac } - outgoing[result.id] = result.response - c.outgoing <- outgoing - // now we are ready to send + + event.Metric(ctx, tag.Started.Of(1)) if err := c.write(ctx, call); err != nil { - // sending failed, we will never get a response, so deliver a fake one - r, _ := NewResponse(result.id, nil, err) - c.incomingResponse(r) + // Sending failed. We will never get a response, so deliver a fake one if it + // wasn't already retired by the connection breaking. + c.updateInFlight(func(s *inFlightState) { + if s.outgoingCalls[ac.id] == ac { + delete(s.outgoingCalls, ac.id) + ac.retire(&Response{ID: id, Error: err}) + } else { + // ac was already retired by the readIncoming goroutine: + // perhaps our write raced with the Read side of the connection breaking. + } + }) } - return result + return ac +} + +type AsyncCall struct { + id ID + ready chan struct{} // closed after response has been set and span has been ended + response *Response + ctx context.Context // for event logging only + endSpan func() // close the tracing span when all processing for the message is complete } // ID used for this call. // This can be used to cancel the call if needed. -func (a *AsyncCall) ID() ID { return a.id } +func (ac *AsyncCall) ID() ID { return ac.id } // IsReady can be used to check if the result is already prepared. // This is guaranteed to return true on a result for which Await has already // returned, or a call that failed to send in the first place. -func (a *AsyncCall) IsReady() bool { +func (ac *AsyncCall) IsReady() bool { select { - case r := <-a.result: - a.result <- r + case <-ac.ready: return true default: return false } } -// Await the results of a Call. +// retire processes the response to the call. +func (ac *AsyncCall) retire(response *Response) { + select { + case <-ac.ready: + panic(fmt.Sprintf("jsonrpc2: retire called twice for ID %v", ac.id)) + default: + } + + ac.response = response + labelStatus(ac.ctx, response.Error) + ac.endSpan() + // Allow the trace context, which may retain a lot of reachable values, + // to be garbage-collected. + ac.ctx, ac.endSpan = nil, nil + + close(ac.ready) +} + +// Await waits for (and decodes) the results of a Call. // The response will be unmarshaled from JSON into the result. -func (a *AsyncCall) Await(ctx context.Context, result interface{}) error { - defer a.endSpan() - var r asyncResult +func (ac *AsyncCall) Await(ctx context.Context, result interface{}) error { select { - case response := <-a.response: - // response just arrived, prepare the result - switch { - case response.Error != nil: - r.err = response.Error - event.Label(ctx, tag.StatusCode.Of("ERROR")) - default: - r.result = response.Result - event.Label(ctx, tag.StatusCode.Of("OK")) - } - case r = <-a.result: - // result already available case <-ctx.Done(): - event.Label(ctx, tag.StatusCode.Of("CANCELLED")) return ctx.Err() + case <-ac.ready: } - // refill the box for the next caller - a.result <- r - // and unpack the result - if r.err != nil { - return r.err + if ac.response.Error != nil { + return ac.response.Error } - if result == nil || len(r.result) == 0 { + if result == nil { return nil } - return json.Unmarshal(r.result, result) + return json.Unmarshal(ac.response.Result, result) } // Respond delivers a response to an incoming Call. // // Respond must be called exactly once for any message for which a handler // returns ErrAsyncResponse. It must not be called for any other message. -func (c *Connection) Respond(id ID, result interface{}, rerr error) error { - pending := <-c.incoming - defer func() { c.incoming <- pending }() - entry, found := pending[id] - if !found { - return nil +func (c *Connection) Respond(id ID, result interface{}, err error) error { + var req *incomingRequest + c.updateInFlight(func(s *inFlightState) { + req = s.incomingByID[id] + }) + if req == nil { + return c.internalErrorf("Request not found for ID %v", id) + } + + if err == ErrAsyncResponse { + // Respond is supposed to supply the asynchronous response, so it would be + // confusing to call Respond with an error that promises to call Respond + // again. + err = c.internalErrorf("Respond called with ErrAsyncResponse for %q", req.Method) } - delete(pending, id) - return c.respond(entry, result, rerr) + return c.processResult("Respond", req, result, err) } -// Cancel is used to cancel an inbound message by ID, it does not cancel -// outgoing messages. -// This is only used inside a message handler that is layering a -// cancellation protocol on top of JSON RPC 2. -// It will not complain if the ID is not a currently active message, and it will -// not cause any messages that have not arrived yet with that ID to be +// Cancel cancels the Context passed to the Handle call for the inbound message +// with the given ID. +// +// Cancel will not complain if the ID is not a currently active message, and it +// will not cause any messages that have not arrived yet with that ID to be // cancelled. func (c *Connection) Cancel(id ID) { - pending := <-c.incoming - defer func() { c.incoming <- pending }() - if entry, found := pending[id]; found && entry.cancel != nil { - entry.cancel() - entry.cancel = nil + var req *incomingRequest + c.updateInFlight(func(s *inFlightState) { + req = s.incomingByID[id] + }) + if req != nil { + req.cancel() } } // Wait blocks until the connection is fully closed, but does not close it. func (c *Connection) Wait() error { - return c.async.wait() + var err error + <-c.done + c.updateInFlight(func(s *inFlightState) { + err = s.closeErr + }) + return err } -// Close can be used to close the underlying stream, and then wait for the connection to -// fully shut down. -// This does not cancel in flight requests, but waits for them to gracefully complete. +// Close stops accepting new requests, waits for in-flight requests and enqueued +// Handle calls to complete, and then closes the underlying stream. +// +// After the start of a Close, notification requests (that lack IDs and do not +// receive responses) will continue to be passed to the Preempter, but calls +// with IDs will receive immediate responses with ErrServerClosing, and no new +// requests (not even notifications!) will be enqueued to the Handler. func (c *Connection) Close() error { - // close the underlying stream - c.closeOnce.Do(func() { - if err := c.closer.Close(); err != nil { - c.async.setError(err) - } - }) - // and then wait for it to cause the connection to close + // Stop handling new requests, and interrupt the reader (by closing the + // connection) as soon as the active requests finish. + c.updateInFlight(func(s *inFlightState) { s.connClosing = true }) + return c.Wait() } // readIncoming collects inbound messages from the reader and delivers them, either responding // to outgoing calls or feeding requests to the queue. -func (c *Connection) readIncoming(ctx context.Context, reader Reader, toQueue chan<- *incoming) (err error) { - defer func() { - // Retire any outgoing requests that were still in flight. - // With the Reader no longer being processed, they necessarily cannot receive a response. - outgoing := <-c.outgoing - close(c.outgoing) // Prevent new outgoing requests, which would deadlock. - for id, response := range outgoing { - response <- &Response{ID: id, Error: err} - } - - close(toQueue) - }() - +func (c *Connection) readIncoming(ctx context.Context, reader Reader, preempter Preempter) { + var err error for { - // get the next message - // no lock is needed, this is the only reader - msg, n, err := reader.Read(ctx) + var ( + msg Message + n int64 + ) + msg, n, err = reader.Read(ctx) if err != nil { - // The stream failed, we cannot continue - if !isClosingError(err) { - c.async.setError(err) - } - return err + break } + switch msg := msg.(type) { case *Request: - entry := &incoming{ - request: msg, - } - // add a span to the context for this request - labels := append(make([]label.Label, 0, 3), // make space for the id if present - tag.Method.Of(msg.Method), - tag.RPCDirection.Of(tag.Inbound), - ) - if msg.IsCall() { - labels = append(labels, tag.RPCID.Of(fmt.Sprintf("%q", msg.ID))) - } - entry.baseCtx, entry.done = event.Start(ctx, msg.Method, labels...) - event.Metric(entry.baseCtx, - tag.Started.Of(1), - tag.ReceivedBytes.Of(n)) - // in theory notifications cannot be cancelled, but we build them a cancel context anyway - entry.handleCtx, entry.cancel = context.WithCancel(entry.baseCtx) - // if the request is a call, add it to the incoming map so it can be - // cancelled by id - if msg.IsCall() { - pending := <-c.incoming - pending[msg.ID] = entry - c.incoming <- pending - } - // send the message to the incoming queue - toQueue <- entry + c.acceptRequest(ctx, msg, n, preempter) + case *Response: - // If method is not set, this should be a response, in which case we must - // have an id to send the response back to the caller. - c.incomingResponse(msg) + c.updateInFlight(func(s *inFlightState) { + if ac, ok := s.outgoingCalls[msg.ID]; ok { + delete(s.outgoingCalls, msg.ID) + ac.retire(msg) + } else { + // TODO: How should we report unexpected responses? + } + }) + + default: + c.internalErrorf("Read returned an unexpected message of type %T", msg) } } + + c.updateInFlight(func(s *inFlightState) { + s.reading = false + s.readErr = err + + // Retire any outgoing requests that were still in flight: with the Reader no + // longer being processed, they necessarily cannot receive a response. + for id, ac := range s.outgoingCalls { + ac.retire(&Response{ID: id, Error: err}) + } + s.outgoingCalls = nil + }) } -func (c *Connection) incomingResponse(msg *Response) { - var response chan<- *Response - if outgoing, ok := <-c.outgoing; ok { - response = outgoing[msg.ID] - delete(outgoing, msg.ID) - c.outgoing <- outgoing +// acceptRequest either handles msg synchronously or enqueues it to be handled +// asynchronously. +func (c *Connection) acceptRequest(ctx context.Context, msg *Request, msgBytes int64, preempter Preempter) { + // Add a span to the context for this request. + labels := append(make([]label.Label, 0, 3), // Make space for the ID if present. + tag.Method.Of(msg.Method), + tag.RPCDirection.Of(tag.Inbound), + ) + if msg.IsCall() { + labels = append(labels, tag.RPCID.Of(fmt.Sprintf("%q", msg.ID))) } - if response != nil { - response <- msg + ctx, endSpan := event.Start(ctx, msg.Method, labels...) + event.Metric(ctx, + tag.Started.Of(1), + tag.ReceivedBytes.Of(msgBytes)) + + // In theory notifications cannot be cancelled, but we build them a cancel + // context anyway. + ctx, cancel := context.WithCancel(ctx) + req := &incomingRequest{ + Request: msg, + ctx: ctx, + cancel: cancel, + endSpan: endSpan, } -} -// manageQueue reads incoming requests, attempts to process them with the preempter, or queue them -// up for normal handling. -func (c *Connection) manageQueue(ctx context.Context, preempter Preempter, fromRead <-chan *incoming, toDeliver chan<- *incoming) { - defer close(toDeliver) - q := []*incoming{} - ok := true - for { - var nextReq *incoming - if len(q) == 0 { - // no messages in the queue - // if we were closing, then we are done - if !ok { + // If the request is a call, add it to the incoming map so it can be + // cancelled (or responded) by ID. + var err error + c.updateInFlight(func(s *inFlightState) { + s.incoming++ + + if req.IsCall() { + if s.incomingByID[req.ID] != nil { + err = fmt.Errorf("%w: request ID %v already in use", ErrInvalidRequest, req.ID) + req.ID = ID{} // Don't misattribute this error to the existing request. return } - // not closing, but nothing in the queue, so just block waiting for a read - nextReq, ok = <-fromRead - } else { - // we have a non empty queue, so pick whichever of reading or delivering - // that we can make progress on - select { - case nextReq, ok = <-fromRead: - case toDeliver <- q[0]: - //TODO: this causes a lot of shuffling, should we use a growing ring buffer? compaction? - q = q[1:] + + if s.incomingByID == nil { + s.incomingByID = make(map[ID]*incomingRequest) } + s.incomingByID[req.ID] = req + + // When shutting down, reject all new Call requests, even if they could + // theoretically be handled by the preempter. The preempter could return + // ErrAsyncResponse, which would increase the amount of work in flight + // when we're trying to ensure that it strictly decreases. + err = s.shuttingDown(ErrServerClosing) } - if nextReq != nil { - // TODO: should we allow to limit the queue size? - var result interface{} - rerr := nextReq.handleCtx.Err() - if rerr == nil { - // only preempt if not already cancelled - result, rerr = preempter.Preempt(nextReq.handleCtx, nextReq.request) - } - switch { - case rerr == ErrNotHandled: - // message not handled, add it to the queue for the main handler - q = append(q, nextReq) - case rerr == ErrAsyncResponse: - // message handled but the response will come later - default: - // anything else means the message is fully handled - c.reply(nextReq, result, rerr) - } + }) + if err != nil { + c.processResult("acceptRequest", req, nil, err) + return + } + + if preempter != nil { + result, err := preempter.Preempt(req.ctx, req.Request) + + if req.IsCall() && errors.Is(err, ErrAsyncResponse) { + // This request will remain in flight until Respond is called for it. + return } + + if !errors.Is(err, ErrNotHandled) { + c.processResult("Preempt", req, result, err) + return + } + } + + c.updateInFlight(func(s *inFlightState) { + // If the connection is shutting down, don't enqueue anything to the + // handler — not even notifications. That ensures that if the handler + // continues to make progress, it will eventually become idle and + // close the connection. + err = s.shuttingDown(ErrServerClosing) + if err != nil { + return + } + + // We enqueue requests that have not been preempted to an unbounded slice. + // Unfortunately, we cannot in general limit the size of the handler + // queue: we have to read every response that comes in on the wire + // (because it may be responding to a request issued by, say, an + // asynchronous handler), and in order to get to that response we have + // to read all of the requests that came in ahead of it. + s.handlerQueue = append(s.handlerQueue, req) + if !s.handlerRunning { + // We start the handleAsync goroutine when it has work to do, and let it + // exit when the queue empties. + // + // Otherwise, in order to synchronize the handler we would need some other + // goroutine (probably readIncoming?) to explicitly wait for handleAsync + // to finish, and that would complicate error reporting: either the error + // report from the goroutine would be blocked on the handler emptying its + // queue (which was tried, and introduced a deadlock detected by + // TestCloseCallRace), or the error would need to be reported separately + // from synchronizing completion. Allowing the handler goroutine to exit + // when idle seems simpler than trying to implement either of those + // alternatives correctly. + s.handlerRunning = true + go c.handleAsync() + } + }) + if err != nil { + c.processResult("acceptRequest", req, nil, err) } } -func (c *Connection) deliverMessages(ctx context.Context, handler Handler, fromQueue <-chan *incoming) { - defer func() { - // Close the underlying ReadWriteCloser if not already closed. We're about - // to mark the Connection as done, so we'd better actually be done! 😅 - // - // TODO(bcmills): This is actually a bit premature, since we may have - // asynchronous handlers still in flight at this point, but it's at least no - // more premature than calling c.async.done at this point (which we were - // already doing). This will get a proper fix in https://go.dev/cl/388134. - c.closeOnce.Do(func() { - if err := c.closer.Close(); err != nil { - c.async.setError(err) +// handleAsync invokes the handler on the requests in the handler queue +// sequentially until the queue is empty. +func (c *Connection) handleAsync() { + for { + var req *incomingRequest + c.updateInFlight(func(s *inFlightState) { + if len(s.handlerQueue) > 0 { + req, s.handlerQueue = s.handlerQueue[0], s.handlerQueue[1:] + } else { + s.handlerRunning = false } }) - - c.async.done() - }() - - for entry := range fromQueue { - // cancel any messages in the queue that we have a pending cancel for - var result interface{} - rerr := entry.handleCtx.Err() - if rerr == nil { - // only deliver if not already cancelled - result, rerr = handler.Handle(entry.handleCtx, entry.request) + if req == nil { + return } - switch { - case rerr == ErrNotHandled: - // message not handled, report it back to the caller as an error - c.reply(entry, nil, fmt.Errorf("%w: %q", ErrMethodNotFound, entry.request.Method)) - case rerr == ErrAsyncResponse: - // message handled but the response will come later - default: - c.reply(entry, result, rerr) + + // Only deliver to the Handler if not already canceled. + if err := req.ctx.Err(); err != nil { + c.updateInFlight(func(s *inFlightState) { + if s.writeErr != nil { + // Assume that req.ctx was canceled due to s.writeErr. + // TODO(#51365): use a Context API to plumb this through req.ctx. + err = fmt.Errorf("%w: %v", ErrServerClosing, s.writeErr) + } + }) + c.processResult("handleAsync", req, nil, err) + continue } + + result, err := c.handler.Handle(req.ctx, req.Request) + c.processResult(c.handler, req, result, err) } } -// reply is used to reply to an incoming request that has just been handled -func (c *Connection) reply(entry *incoming, result interface{}, rerr error) { - if entry.request.IsCall() { - // we have a call finishing, remove it from the incoming map - pending := <-c.incoming - defer func() { c.incoming <- pending }() - delete(pending, entry.request.ID) +// processResult processes the result of a request and, if appropriate, sends a response. +func (c *Connection) processResult(from interface{}, req *incomingRequest, result interface{}, err error) error { + switch err { + case ErrAsyncResponse: + if !req.IsCall() { + return c.internalErrorf("%#v returned ErrAsyncResponse for a %q Request without an ID", from, req.Method) + } + return nil // This request is still in flight, so don't record the result yet. + case ErrNotHandled, ErrMethodNotFound: + // Add detail describing the unhandled method. + err = fmt.Errorf("%w: %q", ErrMethodNotFound, req.Method) } - if err := c.respond(entry, result, rerr); err != nil { - // no way to propagate this error - //TODO: should we do more than just log it? - event.Error(entry.baseCtx, "jsonrpc2 message delivery failed", err) + + if req.endSpan == nil { + return c.internalErrorf("%#v produced a duplicate %q Response", from, req.Method) } -} -// respond sends a response. -// This is the code shared between reply and SendResponse. -func (c *Connection) respond(entry *incoming, result interface{}, rerr error) error { - var err error - if entry.request.IsCall() { - // send the response - if result == nil && rerr == nil { - // call with no response, send an error anyway - rerr = fmt.Errorf("%w: %q produced no response", ErrInternal, entry.request.Method) + if result != nil && err != nil { + c.internalErrorf("%#v returned a non-nil result with a non-nil error for %s:\n%v\n%#v", from, req.Method, err, result) + result = nil // Discard the spurious result and respond with err. + } + + if req.IsCall() { + if result == nil && err == nil { + err = c.internalErrorf("%#v returned a nil result and nil error for a %q Request that requires a Response", from, req.Method) } - var response *Response - response, err = NewResponse(entry.request.ID, result, rerr) - if err == nil { - // we write the response with the base context, in case the message was cancelled - err = c.write(entry.baseCtx, response) + + response, respErr := NewResponse(req.ID, result, err) + + // The caller could theoretically reuse the request's ID as soon as we've + // sent the response, so ensure that it is removed from the incoming map + // before sending. + c.updateInFlight(func(s *inFlightState) { + delete(s.incomingByID, req.ID) + }) + if respErr == nil { + writeErr := c.write(notDone{req.ctx}, response) + if err == nil { + err = writeErr + } + } else { + err = c.internalErrorf("%#v returned a malformed result for %q: %w", from, req.Method, respErr) } - } else { - switch { - case rerr != nil: - // notification failed - err = fmt.Errorf("%w: %q notification failed: %v", ErrInternal, entry.request.Method, rerr) - rerr = nil - case result != nil: - //notification produced a response, which is an error - err = fmt.Errorf("%w: %q produced unwanted response", ErrInternal, entry.request.Method) - default: - // normal notification finish + } else { // req is a notification + if result != nil { + err = c.internalErrorf("%#v returned a non-nil result for a %q Request without an ID", from, req.Method) + } else if err != nil { + err = fmt.Errorf("%w: %q notification failed: %v", ErrInternal, req.Method, err) + } + if err != nil { + // TODO: can/should we do anything with this error beyond writing it to the event log? + // (Is this the right label to attach to the log?) + event.Label(req.ctx, keys.Err.Of(err)) } } - switch { - case rerr != nil || err != nil: - event.Label(entry.baseCtx, tag.StatusCode.Of("ERROR")) - default: - event.Label(entry.baseCtx, tag.StatusCode.Of("OK")) - } - // and just to be clean, invoke and clear the cancel if needed - if entry.cancel != nil { - entry.cancel() - entry.cancel = nil - } - // mark the entire request processing as done - entry.done() - return err + + labelStatus(req.ctx, err) + + // Cancel the request and finalize the event span to free any associated resources. + req.cancel() + req.endSpan() + req.endSpan = nil + c.updateInFlight(func(s *inFlightState) { + if s.incoming == 0 { + panic("jsonrpc2_v2: processResult called when incoming count is already zero") + } + s.incoming-- + }) + return nil } // write is used by all things that write outgoing messages, including replies. @@ -540,5 +755,58 @@ func (c *Connection) write(ctx context.Context, msg Message) error { defer func() { c.writer <- writer }() n, err := writer.Write(ctx, msg) event.Metric(ctx, tag.SentBytes.Of(n)) + + if err != nil && ctx.Err() == nil { + // The call to Write failed, and since ctx.Err() is nil we can't attribute + // the failure (even indirectly) to Context cancellation. The writer appears + // to be broken, and future writes are likely to also fail. + // + // If the read side of the connection is also broken, we might not even be + // able to receive cancellation notifications. Since we can't reliably write + // the results of incoming calls and can't receive explicit cancellations, + // cancel the calls now. + c.updateInFlight(func(s *inFlightState) { + if s.writeErr == nil { + s.writeErr = err + for _, r := range s.incomingByID { + r.cancel() + } + } + }) + } + return err } + +// internalErrorf reports an internal error. By default it panics, but if +// c.onInternalError is non-nil it instead calls that and returns an error +// wrapping ErrInternal. +func (c *Connection) internalErrorf(format string, args ...interface{}) error { + err := fmt.Errorf(format, args...) + if c.onInternalError == nil { + panic("jsonrpc2: " + err.Error()) + } + c.onInternalError(err) + + return fmt.Errorf("%w: %v", ErrInternal, err) +} + +// labelStatus labels the status of the event in ctx based on whether err is nil. +func labelStatus(ctx context.Context, err error) { + if err == nil { + event.Label(ctx, tag.StatusCode.Of("OK")) + } else { + event.Label(ctx, tag.StatusCode.Of("ERROR")) + } +} + +// notDone is a context.Context wrapper that returns a nil Done channel. +type notDone struct{ ctx context.Context } + +func (ic notDone) Value(key interface{}) interface{} { + return ic.ctx.Value(key) +} + +func (notDone) Done() <-chan struct{} { return nil } +func (notDone) Err() error { return nil } +func (notDone) Deadline() (time.Time, bool) { return time.Time{}, false } diff --git a/internal/jsonrpc2_v2/frame.go b/internal/jsonrpc2_v2/frame.go index b2b7dc1a172..e4248328132 100644 --- a/internal/jsonrpc2_v2/frame.go +++ b/internal/jsonrpc2_v2/frame.go @@ -120,6 +120,12 @@ func (r *headerReader) Read(ctx context.Context) (Message, int64, error) { line, err := r.in.ReadString('\n') total += int64(len(line)) if err != nil { + if err == io.EOF { + if total == 0 { + return nil, 0, io.EOF + } + err = io.ErrUnexpectedEOF + } return nil, total, fmt.Errorf("failed reading header line: %w", err) } line = strings.TrimSpace(line) diff --git a/internal/jsonrpc2_v2/jsonrpc2_test.go b/internal/jsonrpc2_v2/jsonrpc2_test.go index b2fa4963b74..dd8d09c8870 100644 --- a/internal/jsonrpc2_v2/jsonrpc2_test.go +++ b/internal/jsonrpc2_v2/jsonrpc2_test.go @@ -136,10 +136,7 @@ func testConnection(t *testing.T, framer jsonrpc2.Framer) { if err != nil { t.Fatal(err) } - server, err := jsonrpc2.Serve(ctx, listener, binder{framer, nil}) - if err != nil { - t.Fatal(err) - } + server := jsonrpc2.NewServer(ctx, listener, binder{framer, nil}) defer func() { listener.Close() server.Wait() @@ -253,7 +250,7 @@ func verifyResults(t *testing.T, method string, results interface{}, expect inte } } -func (b binder) Bind(ctx context.Context, conn *jsonrpc2.Connection) (jsonrpc2.ConnectionOptions, error) { +func (b binder) Bind(ctx context.Context, conn *jsonrpc2.Connection) jsonrpc2.ConnectionOptions { h := &handler{ conn: conn, waiters: make(chan map[string]chan struct{}, 1), @@ -267,7 +264,7 @@ func (b binder) Bind(ctx context.Context, conn *jsonrpc2.Connection) (jsonrpc2.C Framer: b.framer, Preempter: h, Handler: h, - }, nil + } } func (h *handler) waiter(name string) chan struct{} { diff --git a/internal/jsonrpc2_v2/net.go b/internal/jsonrpc2_v2/net.go index f1e2b0c7b36..15d0aea3af0 100644 --- a/internal/jsonrpc2_v2/net.go +++ b/internal/jsonrpc2_v2/net.go @@ -9,7 +9,6 @@ import ( "io" "net" "os" - "time" ) // This file contains implementations of the transport primitives that use the standard network @@ -36,7 +35,7 @@ type netListener struct { } // Accept blocks waiting for an incoming connection to the listener. -func (l *netListener) Accept(ctx context.Context) (io.ReadWriteCloser, error) { +func (l *netListener) Accept(context.Context) (io.ReadWriteCloser, error) { return l.net.Accept() } @@ -56,9 +55,7 @@ func (l *netListener) Close() error { // Dialer returns a dialer that can be used to connect to the listener. func (l *netListener) Dialer() Dialer { - return NetDialer(l.net.Addr().Network(), l.net.Addr().String(), net.Dialer{ - Timeout: 5 * time.Second, - }) + return NetDialer(l.net.Addr().Network(), l.net.Addr().String(), net.Dialer{}) } // NetDialer returns a Dialer using the supplied standard network dialer. @@ -98,15 +95,19 @@ type netPiper struct { } // Accept blocks waiting for an incoming connection to the listener. -func (l *netPiper) Accept(ctx context.Context) (io.ReadWriteCloser, error) { - // block until we have a listener, or are closed or cancelled +func (l *netPiper) Accept(context.Context) (io.ReadWriteCloser, error) { + // Block until the pipe is dialed or the listener is closed, + // preferring the latter if already closed at the start of Accept. + select { + case <-l.done: + return nil, errClosed + default: + } select { case rwc := <-l.dialed: return rwc, nil case <-l.done: - return nil, io.EOF - case <-ctx.Done(): - return nil, ctx.Err() + return nil, errClosed } } @@ -124,6 +125,14 @@ func (l *netPiper) Dialer() Dialer { func (l *netPiper) Dial(ctx context.Context) (io.ReadWriteCloser, error) { client, server := net.Pipe() - l.dialed <- server - return client, nil + + select { + case l.dialed <- server: + return client, nil + + case <-l.done: + client.Close() + server.Close() + return nil, errClosed + } } diff --git a/internal/jsonrpc2_v2/serve.go b/internal/jsonrpc2_v2/serve.go index 5ffce681e91..7df785655d7 100644 --- a/internal/jsonrpc2_v2/serve.go +++ b/internal/jsonrpc2_v2/serve.go @@ -6,13 +6,11 @@ package jsonrpc2 import ( "context" - "errors" "fmt" "io" "runtime" - "strings" "sync" - "syscall" + "sync/atomic" "time" ) @@ -43,35 +41,43 @@ type Server struct { listener Listener binder Binder async *async + + shutdownOnce sync.Once + closing int32 // atomic: set to nonzero when Shutdown is called } // Dial uses the dialer to make a new connection, wraps the returned // reader and writer using the framer to make a stream, and then builds // a connection on top of that stream using the binder. +// +// The returned Connection will operate independently using the Preempter and/or +// Handler provided by the Binder, and will release its own resources when the +// connection is broken, but the caller may Close it earlier to stop accepting +// (or sending) new requests. func Dial(ctx context.Context, dialer Dialer, binder Binder) (*Connection, error) { // dial a server rwc, err := dialer.Dial(ctx) if err != nil { return nil, err } - return newConnection(ctx, rwc, binder) + return newConnection(ctx, rwc, binder, nil), nil } -// Serve starts a new server listening for incoming connections and returns +// NewServer starts a new server listening for incoming connections and returns // it. // This returns a fully running and connected server, it does not block on // the listener. // You can call Wait to block on the server, or Shutdown to get the sever to // terminate gracefully. // To notice incoming connections, use an intercepting Binder. -func Serve(ctx context.Context, listener Listener, binder Binder) (*Server, error) { +func NewServer(ctx context.Context, listener Listener, binder Binder) *Server { server := &Server{ listener: listener, binder: binder, async: newAsync(), } go server.run(ctx) - return server, nil + return server } // Wait returns only when the server has shut down. @@ -79,96 +85,39 @@ func (s *Server) Wait() error { return s.async.wait() } +// Shutdown informs the server to stop accepting new connections. +func (s *Server) Shutdown() { + s.shutdownOnce.Do(func() { + atomic.StoreInt32(&s.closing, 1) + s.listener.Close() + }) +} + // run accepts incoming connections from the listener, // If IdleTimeout is non-zero, run exits after there are no clients for this // duration, otherwise it exits only on error. func (s *Server) run(ctx context.Context) { defer s.async.done() - var activeConns []*Connection + + var activeConns sync.WaitGroup for { - // we never close the accepted connection, we rely on the other end - // closing or the socket closing itself naturally rwc, err := s.listener.Accept(ctx) if err != nil { - if !isClosingError(err) { + // Only Shutdown closes the listener. If we get an error after Shutdown is + // called, assume that that was the cause and don't report the error; + // otherwise, report the error in case it is unexpected. + if atomic.LoadInt32(&s.closing) == 0 { s.async.setError(err) } - // we are done generating new connections for good + // We are done generating new connections for good. break } - // see if any connections were closed while we were waiting - activeConns = onlyActive(activeConns) - - // a new inbound connection, - conn, err := newConnection(ctx, rwc, s.binder) - if err != nil { - if !isClosingError(err) { - s.async.setError(err) - } - continue - } - activeConns = append(activeConns, conn) - } - - // wait for all active conns to finish - for _, c := range activeConns { - c.Wait() + // A new inbound connection. + activeConns.Add(1) + _ = newConnection(ctx, rwc, s.binder, activeConns.Done) // unregisters itself when done } -} - -func onlyActive(conns []*Connection) []*Connection { - i := 0 - for _, c := range conns { - if !c.async.isDone() { - conns[i] = c - i++ - } - } - // trim the slice down - return conns[:i] -} - -// isClosingError reports if the error occurs normally during the process of -// closing a network connection. It uses imperfect heuristics that err on the -// side of false negatives, and should not be used for anything critical. -func isClosingError(err error) bool { - if err == nil { - return false - } - // Fully unwrap the error, so the following tests work. - for wrapped := err; wrapped != nil; wrapped = errors.Unwrap(err) { - err = wrapped - } - - // Was it based on an EOF error? - if err == io.EOF { - return true - } - - // Was it based on a closed pipe? - if err == io.ErrClosedPipe { - return true - } - - // Per https://github.com/golang/go/issues/4373, this error string should not - // change. This is not ideal, but since the worst that could happen here is - // some superfluous logging, it is acceptable. - if err.Error() == "use of closed network connection" { - return true - } - - if runtime.GOOS == "plan9" { - // Error reading from a closed connection. - if err == syscall.EINVAL { - return true - } - // Error trying to accept a new connection from a closed listener. - if strings.HasSuffix(err.Error(), " listen hungup") { - return true - } - } - return false + activeConns.Wait() } // NewIdleListener wraps a listener with an idle timeout. @@ -206,10 +155,6 @@ type idleListener struct { func (l *idleListener) Accept(ctx context.Context) (io.ReadWriteCloser, error) { rwc, err := l.wrapped.Accept(ctx) - if err != nil && !isClosingError(err) { - return nil, err - } - select { case n, ok := <-l.active: if err != nil { @@ -234,13 +179,20 @@ func (l *idleListener) Accept(ctx context.Context) (io.ReadWriteCloser, error) { // active and closed due to idleness, which would be contradictory and // confusing. Close the connection and pretend that it never happened. rwc.Close() + } else { + // In theory the timeout could have raced with an unrelated error return + // from Accept. However, ErrIdleTimeout is arguably still valid (since we + // would have closed due to the timeout independent of the error), and the + // harm from returning a spurious ErrIdleTimeout is negliglible anyway. } return nil, ErrIdleTimeout case timer := <-l.idleTimer: if err != nil { - // The idle timer hasn't run yet, so err can't be ErrIdleTimeout. - // Leave the idle timer as it was and return whatever error we got. + // The idle timer doesn't run until it receives itself from the idleTimer + // channel, so it can't have called l.wrapped.Close yet and thus err can't + // be ErrIdleTimeout. Leave the idle timer as it was and return whatever + // error we got. l.idleTimer <- timer return nil, err } diff --git a/internal/jsonrpc2_v2/serve_go116.go b/internal/jsonrpc2_v2/serve_go116.go new file mode 100644 index 00000000000..29549f1059d --- /dev/null +++ b/internal/jsonrpc2_v2/serve_go116.go @@ -0,0 +1,19 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.16 +// +build go1.16 + +package jsonrpc2 + +import ( + "errors" + "net" +) + +var errClosed = net.ErrClosed + +func isErrClosed(err error) bool { + return errors.Is(err, errClosed) +} diff --git a/internal/jsonrpc2_v2/serve_pre116.go b/internal/jsonrpc2_v2/serve_pre116.go new file mode 100644 index 00000000000..14afa834962 --- /dev/null +++ b/internal/jsonrpc2_v2/serve_pre116.go @@ -0,0 +1,30 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !go1.16 +// +build !go1.16 + +package jsonrpc2 + +import ( + "errors" + "strings" +) + +// errClosed is an error with the same string as net.ErrClosed, +// which was added in Go 1.16. +var errClosed = errors.New("use of closed network connection") + +// isErrClosed reports whether err ends in the same string as errClosed. +func isErrClosed(err error) bool { + // As of Go 1.16, this could be 'errors.Is(err, net.ErrClosing)', but + // unfortunately gopls still requires compatiblity with + // (otherwise-unsupported) older Go versions. + // + // In the meantime, this error strirng has not changed on any supported Go + // version, and is not expected to change in the future. + // This is not ideal, but since the worst that could happen here is some + // superfluous logging, it is acceptable. + return strings.HasSuffix(err.Error(), "use of closed network connection") +} diff --git a/internal/jsonrpc2_v2/serve_test.go b/internal/jsonrpc2_v2/serve_test.go index f0c27a8be56..88ac66b7e66 100644 --- a/internal/jsonrpc2_v2/serve_test.go +++ b/internal/jsonrpc2_v2/serve_test.go @@ -41,10 +41,7 @@ func TestIdleTimeout(t *testing.T) { listener = jsonrpc2.NewIdleListener(d, listener) defer listener.Close() - server, err := jsonrpc2.Serve(ctx, listener, jsonrpc2.ConnectionOptions{}) - if err != nil { - t.Fatal(err) - } + server := jsonrpc2.NewServer(ctx, listener, jsonrpc2.ConnectionOptions{}) // Exercise some connection/disconnection patterns, and then assert that when // our timer fires, the server exits. @@ -69,15 +66,25 @@ func TestIdleTimeout(t *testing.T) { return false } + // Since conn1 was successfully accepted and remains open, the server is + // definitely non-idle. Dialing another simultaneous connection should + // succeed. conn2, err := jsonrpc2.Dial(ctx, listener.Dialer(), jsonrpc2.ConnectionOptions{}) if err != nil { conn1.Close() - if since := time.Since(idleStart); since < d { - t.Fatalf("conn2 failed to connect while non-idle: %v", err) - } - t.Log("jsonrpc2.Dial:", err) + t.Fatalf("conn2 failed to connect while non-idle after %v: %v", time.Since(idleStart), err) return false } + // Ensure that conn2 is also accepted on the server side before we close + // conn1. Otherwise, the connection can appear idle if the server processes + // the closure of conn1 and the idle timeout before it finally notices conn2 + // in the accept queue. + // (That failure mode may explain the failure noted in + // https://go.dev/issue/49387#issuecomment-1303979877.) + ac = conn2.Call(ctx, "ping", nil) + if err := ac.Await(ctx, nil); !errors.Is(err, jsonrpc2.ErrMethodNotFound) { + t.Fatalf("conn2 broken while non-idle after %v: %v", time.Since(idleStart), err) + } if err := conn1.Close(); err != nil { t.Fatalf("conn1.Close failed with error: %v", err) @@ -187,12 +194,9 @@ func TestServe(t *testing.T) { } func newFake(t *testing.T, ctx context.Context, l jsonrpc2.Listener) (*jsonrpc2.Connection, func(), error) { - server, err := jsonrpc2.Serve(ctx, l, jsonrpc2.ConnectionOptions{ + server := jsonrpc2.NewServer(ctx, l, jsonrpc2.ConnectionOptions{ Handler: fakeHandler{}, }) - if err != nil { - return nil, nil, err - } client, err := jsonrpc2.Dial(ctx, l.Dialer(), @@ -230,7 +234,7 @@ func TestIdleListenerAcceptCloseRace(t *testing.T) { watchdog := time.Duration(n) * 1000 * time.Millisecond timer := time.AfterFunc(watchdog, func() { debug.SetTraceback("all") - panic(fmt.Sprintf("TestAcceptCloseRace deadlocked after %v", watchdog)) + panic(fmt.Sprintf("%s deadlocked after %v", t.Name(), watchdog)) }) defer timer.Stop() @@ -261,3 +265,80 @@ func TestIdleListenerAcceptCloseRace(t *testing.T) { <-done } } + +// TestCloseCallRace checks for a race resulting in a deadlock when a Call on +// one side of the connection races with a Close (or otherwise broken +// connection) initiated from the other side. +// +// (The Call method was waiting for a result from the Read goroutine to +// determine which error value to return, but the Read goroutine was waiting for +// in-flight calls to complete before reporting that result.) +func TestCloseCallRace(t *testing.T) { + ctx := context.Background() + n := 10 + + watchdog := time.Duration(n) * 1000 * time.Millisecond + timer := time.AfterFunc(watchdog, func() { + debug.SetTraceback("all") + panic(fmt.Sprintf("%s deadlocked after %v", t.Name(), watchdog)) + }) + defer timer.Stop() + + for ; n > 0; n-- { + listener, err := jsonrpc2.NetPipeListener(ctx) + if err != nil { + t.Fatal(err) + } + + pokec := make(chan *jsonrpc2.AsyncCall, 1) + + s := jsonrpc2.NewServer(ctx, listener, jsonrpc2.BinderFunc(func(_ context.Context, srvConn *jsonrpc2.Connection) jsonrpc2.ConnectionOptions { + h := jsonrpc2.HandlerFunc(func(ctx context.Context, _ *jsonrpc2.Request) (interface{}, error) { + // Start a concurrent call from the server to the client. + // The point of this test is to ensure this doesn't deadlock + // if the client shuts down the connection concurrently. + // + // The racing Call may or may not receive a response: it should get a + // response if it is sent before the client closes the connection, and + // it should fail with some kind of "connection closed" error otherwise. + go func() { + pokec <- srvConn.Call(ctx, "poke", nil) + }() + + return &msg{"pong"}, nil + }) + return jsonrpc2.ConnectionOptions{Handler: h} + })) + + dialConn, err := jsonrpc2.Dial(ctx, listener.Dialer(), jsonrpc2.ConnectionOptions{}) + if err != nil { + listener.Close() + s.Wait() + t.Fatal(err) + } + + // Calling any method on the server should provoke it to asynchronously call + // us back. While it is starting that call, we will close the connection. + if err := dialConn.Call(ctx, "ping", nil).Await(ctx, nil); err != nil { + t.Error(err) + } + if err := dialConn.Close(); err != nil { + t.Error(err) + } + + // Ensure that the Call on the server side did not block forever when the + // connection closed. + pokeCall := <-pokec + if err := pokeCall.Await(ctx, nil); err == nil { + t.Errorf("unexpected nil error from server-initited call") + } else if errors.Is(err, jsonrpc2.ErrMethodNotFound) { + // The call completed before the Close reached the handler. + } else { + // The error was something else. + t.Logf("server-initiated call completed with expected error: %v", err) + } + + listener.Close() + s.Wait() + } +} diff --git a/internal/loopclosure/main.go b/internal/loopclosure/main.go deleted file mode 100644 index 03238edae13..00000000000 --- a/internal/loopclosure/main.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// The loopclosure command applies the golang.org/x/tools/go/analysis/passes/loopclosure -// analysis to the specified packages of Go source code. It enables -// experimental checking of parallel subtests. -// -// TODO: Once the parallel subtest experiment is complete, this can be made -// public at go/analysis/passes/loopclosure/cmd, or deleted. -package main - -import ( - "golang.org/x/tools/go/analysis/passes/loopclosure" - "golang.org/x/tools/go/analysis/singlechecker" - "golang.org/x/tools/internal/analysisinternal" -) - -func main() { - analysisinternal.LoopclosureParallelSubtests = true - singlechecker.Main(loopclosure.Analyzer) -} diff --git a/internal/persistent/map_test.go b/internal/persistent/map_test.go index 1c413d78fa7..9f89a1d300c 100644 --- a/internal/persistent/map_test.go +++ b/internal/persistent/map_test.go @@ -19,14 +19,15 @@ type mapEntry struct { type validatedMap struct { impl *Map - expected map[int]int - deleted map[mapEntry]struct{} - seen map[mapEntry]struct{} + expected map[int]int // current key-value mapping. + deleted map[mapEntry]int // maps deleted entries to their clock time of last deletion + seen map[mapEntry]int // maps seen entries to their clock time of last insertion + clock int } func TestSimpleMap(t *testing.T) { - deletedEntries := make(map[mapEntry]struct{}) - seenEntries := make(map[mapEntry]struct{}) + deletedEntries := make(map[mapEntry]int) + seenEntries := make(map[mapEntry]int) m1 := &validatedMap{ impl: NewMap(func(a, b interface{}) bool { @@ -43,7 +44,7 @@ func TestSimpleMap(t *testing.T) { validateRef(t, m1, m3) m3.destroy() - assertSameMap(t, deletedEntries, map[mapEntry]struct{}{ + assertSameMap(t, entrySet(deletedEntries), map[mapEntry]struct{}{ {key: 8, value: 8}: {}, }) @@ -59,7 +60,7 @@ func TestSimpleMap(t *testing.T) { m1.set(t, 6, 6) validateRef(t, m1) - assertSameMap(t, deletedEntries, map[mapEntry]struct{}{ + assertSameMap(t, entrySet(deletedEntries), map[mapEntry]struct{}{ {key: 2, value: 2}: {}, {key: 8, value: 8}: {}, }) @@ -98,7 +99,7 @@ func TestSimpleMap(t *testing.T) { m1.destroy() - assertSameMap(t, deletedEntries, map[mapEntry]struct{}{ + assertSameMap(t, entrySet(deletedEntries), map[mapEntry]struct{}{ {key: 2, value: 2}: {}, {key: 6, value: 60}: {}, {key: 8, value: 8}: {}, @@ -114,12 +115,12 @@ func TestSimpleMap(t *testing.T) { m2.destroy() - assertSameMap(t, seenEntries, deletedEntries) + assertSameMap(t, entrySet(seenEntries), entrySet(deletedEntries)) } func TestRandomMap(t *testing.T) { - deletedEntries := make(map[mapEntry]struct{}) - seenEntries := make(map[mapEntry]struct{}) + deletedEntries := make(map[mapEntry]int) + seenEntries := make(map[mapEntry]int) m := &validatedMap{ impl: NewMap(func(a, b interface{}) bool { @@ -132,7 +133,7 @@ func TestRandomMap(t *testing.T) { keys := make([]int, 0, 1000) for i := 0; i < 1000; i++ { - key := rand.Int() + key := rand.Intn(10000) m.set(t, key, key) keys = append(keys, key) @@ -148,12 +149,20 @@ func TestRandomMap(t *testing.T) { } m.destroy() - assertSameMap(t, seenEntries, deletedEntries) + assertSameMap(t, entrySet(seenEntries), entrySet(deletedEntries)) +} + +func entrySet(m map[mapEntry]int) map[mapEntry]struct{} { + set := make(map[mapEntry]struct{}) + for k := range m { + set[k] = struct{}{} + } + return set } func TestUpdate(t *testing.T) { - deletedEntries := make(map[mapEntry]struct{}) - seenEntries := make(map[mapEntry]struct{}) + deletedEntries := make(map[mapEntry]int) + seenEntries := make(map[mapEntry]int) m1 := &validatedMap{ impl: NewMap(func(a, b interface{}) bool { @@ -173,15 +182,7 @@ func TestUpdate(t *testing.T) { m1.destroy() m2.destroy() - assertSameMap(t, seenEntries, deletedEntries) -} - -func (vm *validatedMap) onDelete(t *testing.T, key, value int) { - entry := mapEntry{key: key, value: value} - if _, ok := vm.deleted[entry]; ok { - t.Fatalf("tried to delete entry twice, key: %d, value: %d", key, value) - } - vm.deleted[entry] = struct{}{} + assertSameMap(t, entrySet(seenEntries), entrySet(deletedEntries)) } func validateRef(t *testing.T, maps ...*validatedMap) { @@ -234,9 +235,12 @@ func (vm *validatedMap) validate(t *testing.T) { validateNode(t, vm.impl.root, vm.impl.less) + // Note: this validation may not make sense if maps were constructed using + // SetAll operations. If this proves to be problematic, remove the clock, + // deleted, and seen fields. for key, value := range vm.expected { entry := mapEntry{key: key, value: value} - if _, ok := vm.deleted[entry]; ok { + if deleteAt := vm.deleted[entry]; deleteAt > vm.seen[entry] { t.Fatalf("entry is deleted prematurely, key: %d, value: %d", key, value) } } @@ -281,6 +285,9 @@ func validateNode(t *testing.T, node *mapNode, less func(a, b interface{}) bool) func (vm *validatedMap) setAll(t *testing.T, other *validatedMap) { vm.impl.SetAll(other.impl) + + // Note: this is buggy because we are not updating vm.clock, vm.deleted, or + // vm.seen. for key, value := range other.expected { vm.expected[key] = value } @@ -288,12 +295,17 @@ func (vm *validatedMap) setAll(t *testing.T, other *validatedMap) { } func (vm *validatedMap) set(t *testing.T, key, value int) { - vm.seen[mapEntry{key: key, value: value}] = struct{}{} + entry := mapEntry{key: key, value: value} + + vm.clock++ + vm.seen[entry] = vm.clock + vm.impl.Set(key, value, func(deletedKey, deletedValue interface{}) { if deletedKey != key || deletedValue != value { t.Fatalf("unexpected passed in deleted entry: %v/%v, expected: %v/%v", deletedKey, deletedValue, key, value) } - vm.onDelete(t, key, value) + // Not safe if closure shared between two validatedMaps. + vm.deleted[entry] = vm.clock }) vm.expected[key] = value vm.validate(t) @@ -305,6 +317,7 @@ func (vm *validatedMap) set(t *testing.T, key, value int) { } func (vm *validatedMap) remove(t *testing.T, key int) { + vm.clock++ vm.impl.Delete(key) delete(vm.expected, key) vm.validate(t) diff --git a/go/internal/pkgbits/codes.go b/internal/pkgbits/codes.go similarity index 100% rename from go/internal/pkgbits/codes.go rename to internal/pkgbits/codes.go diff --git a/go/internal/pkgbits/decoder.go b/internal/pkgbits/decoder.go similarity index 100% rename from go/internal/pkgbits/decoder.go rename to internal/pkgbits/decoder.go diff --git a/go/internal/pkgbits/doc.go b/internal/pkgbits/doc.go similarity index 100% rename from go/internal/pkgbits/doc.go rename to internal/pkgbits/doc.go diff --git a/go/internal/pkgbits/encoder.go b/internal/pkgbits/encoder.go similarity index 100% rename from go/internal/pkgbits/encoder.go rename to internal/pkgbits/encoder.go diff --git a/go/internal/pkgbits/flags.go b/internal/pkgbits/flags.go similarity index 100% rename from go/internal/pkgbits/flags.go rename to internal/pkgbits/flags.go diff --git a/go/internal/pkgbits/frames_go1.go b/internal/pkgbits/frames_go1.go similarity index 100% rename from go/internal/pkgbits/frames_go1.go rename to internal/pkgbits/frames_go1.go diff --git a/go/internal/pkgbits/frames_go17.go b/internal/pkgbits/frames_go17.go similarity index 100% rename from go/internal/pkgbits/frames_go17.go rename to internal/pkgbits/frames_go17.go diff --git a/go/internal/pkgbits/reloc.go b/internal/pkgbits/reloc.go similarity index 100% rename from go/internal/pkgbits/reloc.go rename to internal/pkgbits/reloc.go diff --git a/go/internal/pkgbits/support.go b/internal/pkgbits/support.go similarity index 100% rename from go/internal/pkgbits/support.go rename to internal/pkgbits/support.go diff --git a/go/internal/pkgbits/sync.go b/internal/pkgbits/sync.go similarity index 100% rename from go/internal/pkgbits/sync.go rename to internal/pkgbits/sync.go diff --git a/go/internal/pkgbits/syncmarker_string.go b/internal/pkgbits/syncmarker_string.go similarity index 100% rename from go/internal/pkgbits/syncmarker_string.go rename to internal/pkgbits/syncmarker_string.go diff --git a/internal/testenv/testenv.go b/internal/testenv/testenv.go index bfadb44be65..f606cb71543 100644 --- a/internal/testenv/testenv.go +++ b/internal/testenv/testenv.go @@ -16,8 +16,11 @@ import ( "runtime/debug" "strings" "sync" + "testing" "time" + "golang.org/x/tools/internal/goroot" + exec "golang.org/x/sys/execabs" ) @@ -329,3 +332,20 @@ func Deadline(t Testing) (time.Time, bool) { } return td.Deadline() } + +// WriteImportcfg writes an importcfg file used by the compiler or linker to +// dstPath containing entries for the packages in std and cmd in addition +// to the package to package file mappings in additionalPackageFiles. +func WriteImportcfg(t testing.TB, dstPath string, additionalPackageFiles map[string]string) { + importcfg, err := goroot.Importcfg() + for k, v := range additionalPackageFiles { + importcfg += fmt.Sprintf("\npackagefile %s=%s", k, v) + } + if err != nil { + t.Fatalf("preparing the importcfg failed: %s", err) + } + ioutil.WriteFile(dstPath, []byte(importcfg), 0655) + if err != nil { + t.Fatalf("writing the importcfg failed: %s", err) + } +}