From a4ed05f16747f9958a65c5c279f69e30b6477c9e Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Thu, 15 Jun 2023 16:49:00 -0400 Subject: [PATCH 01/26] cmd/digraph: add "to dot" to emit Graphviz dot This CL adds "digraph to dot", which prints the graph in Graphviz dot format, which can then be piped into dot. Change-Id: I8209d9038bb4e1df1ca2ced427ce14f6df8051db Reviewed-on: https://go-review.googlesource.com/c/tools/+/503437 TryBot-Result: Gopher Robot Reviewed-by: Alan Donovan gopls-CI: kokoro Run-TryBot: Austin Clements --- cmd/digraph/digraph.go | 45 ++++++++++++++++++++++++++++++++----- cmd/digraph/digraph_test.go | 25 +++++++++++++++++++++ 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/cmd/digraph/digraph.go b/cmd/digraph/digraph.go index 93130a018ce..4d033e1ccdb 100644 --- a/cmd/digraph/digraph.go +++ b/cmd/digraph/digraph.go @@ -37,6 +37,8 @@ The support commands are: the set of nodes strongly connected to the specified one focus the subgraph containing all directed paths that pass through the specified node + to dot + print the graph in Graphviz dot format (other formats may be supported in the future) Input format: @@ -81,6 +83,11 @@ Show which packages in x/tools depend, perhaps indirectly, on the callgraph pack $ go list -f '{{.ImportPath}} {{join .Imports " "}}' -deps golang.org/x/tools/... | digraph reverse golang.org/x/tools/go/callgraph +Visualize the package dependency graph of the current package: + + $ go list -f '{{.ImportPath}} {{join .Imports " "}}' -deps | + digraph to dot | dot -Tpng -o x.png + Using a module graph produced by go mod, show all dependencies of the current module: $ go mod graph | digraph forward $(go list -m) @@ -137,6 +144,8 @@ The support commands are: the set of nodes nodes strongly connected to the specified one focus the subgraph containing all directed paths that pass through the specified node + to dot + print the graph in Graphviz dot format (other formats may be supported in the future) `) os.Exit(2) } @@ -207,6 +216,14 @@ func (g graph) addEdges(from string, to ...string) { } } +func (g graph) nodelist() nodelist { + nodes := make(nodeset) + for node := range g { + nodes[node] = true + } + return nodes.sort() +} + func (g graph) reachableFrom(roots nodeset) nodeset { seen := make(nodeset) var visit func(node string) @@ -356,6 +373,20 @@ func (g graph) somepath(from, to string) error { return nil } +func (g graph) toDot(w *bytes.Buffer) { + fmt.Fprintln(w, "digraph {") + for _, src := range g.nodelist() { + for _, dst := range g[src].sort() { + // Dot's quoting rules appear to align with Go's for escString, + // which is the syntax of node IDs. Labels require significantly + // more quoting, but that appears not to be necessary if the node ID + // is implicitly used as the label. + fmt.Fprintf(w, "\t%q -> %q;\n", src, dst) + } + } + fmt.Fprintln(w, "}") +} + func parse(rd io.Reader) (graph, error) { g := make(graph) @@ -404,11 +435,7 @@ func digraph(cmd string, args []string) error { if len(args) != 0 { return fmt.Errorf("usage: digraph nodes") } - nodes := make(nodeset) - for node := range g { - nodes[node] = true - } - nodes.sort().println("\n") + g.nodelist().println("\n") case "degree": if len(args) != 0 { @@ -563,6 +590,14 @@ func digraph(cmd string, args []string) error { sort.Strings(edgesSorted) fmt.Fprintln(stdout, strings.Join(edgesSorted, "\n")) + case "to": + if len(args) != 1 || args[0] != "dot" { + return fmt.Errorf("usage: digraph to dot") + } + var b bytes.Buffer + g.toDot(&b) + stdout.Write(b.Bytes()) + default: return fmt.Errorf("no such command %q", cmd) } diff --git a/cmd/digraph/digraph_test.go b/cmd/digraph/digraph_test.go index 60b8e75eb72..4d238d54b73 100644 --- a/cmd/digraph/digraph_test.go +++ b/cmd/digraph/digraph_test.go @@ -6,6 +6,7 @@ package main import ( "bytes" "fmt" + "io" "reflect" "sort" "strings" @@ -346,3 +347,27 @@ func TestFocus(t *testing.T) { }) } } + +func TestToDot(t *testing.T) { + in := `a b c +b "d\"\\d" +c "d\"\\d"` + want := `digraph { + "a" -> "b"; + "a" -> "c"; + "b" -> "d\"\\d"; + "c" -> "d\"\\d"; +} +` + defer func(in io.Reader, out io.Writer) { stdin, stdout = in, out }(stdin, stdout) + stdin = strings.NewReader(in) + stdout = new(bytes.Buffer) + if err := digraph("to", []string{"dot"}); err != nil { + t.Fatal(err) + } + got := stdout.(fmt.Stringer).String() + if got != want { + t.Errorf("digraph(to, dot) = got %q, want %q", got, want) + } + +} From d1a388b0ec90c90a7127e3af2612f5ef16c4aee8 Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Tue, 20 Jun 2023 12:22:24 -0400 Subject: [PATCH 02/26] internal/gcimporter: avoid a crash when exporting a struct with no pkg This is an arbitrary fix to avoid a crash while exporting an empty struct with no ambient package. Unfortunately, the struct encoding assumes that there is an implied package for structs, which at least in invalid code may not be the case. Fixes golang/go#60891 Change-Id: I8acd591ac15cdc1770d615fdca57dcb7bb9b7651 Reviewed-on: https://go-review.googlesource.com/c/tools/+/504556 TryBot-Result: Gopher Robot Run-TryBot: Robert Findley gopls-CI: kokoro Reviewed-by: Alan Donovan --- internal/gcimporter/gcimporter_test.go | 3 +++ internal/gcimporter/iexport.go | 19 ++++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/internal/gcimporter/gcimporter_test.go b/internal/gcimporter/gcimporter_test.go index e13cfd9219a..6346885a58f 100644 --- a/internal/gcimporter/gcimporter_test.go +++ b/internal/gcimporter/gcimporter_test.go @@ -809,6 +809,9 @@ func TestExportInvalid(t *testing.T) { // It must be possible to export a constant with unknown kind, even if its // type is known. {"issue 60605", `package p; const EPSILON float64 = 1e-`, "EPSILON"}, + + // We must not crash when exporting a struct with unknown package. + {"issue 60891", `package p; type I[P any] int; const C I[struct{}] = 42`, "C"}, } for _, test := range tests { diff --git a/internal/gcimporter/iexport.go b/internal/gcimporter/iexport.go index 3fc7989c083..db9842706c9 100644 --- a/internal/gcimporter/iexport.go +++ b/internal/gcimporter/iexport.go @@ -673,6 +673,9 @@ func (w *exportWriter) qualifiedType(obj *types.TypeName) { w.pkg(obj.Pkg()) } +// TODO(rfindley): what does 'pkg' even mean here? It would be better to pass +// it in explicitly into signatures and structs that may use it for +// constructing fields. func (w *exportWriter) typ(t types.Type, pkg *types.Package) { w.data.uint64(w.p.typOff(t, pkg)) } @@ -773,7 +776,21 @@ func (w *exportWriter) doTyp(t types.Type, pkg *types.Package) { if n > 0 { w.setPkg(t.Field(0).Pkg(), true) // qualifying package for field objects } else { - w.setPkg(pkg, true) + // TODO(rfindley): improve this very hacky logic. + // + // The importer expects a package to be set for all struct types, even + // those with no fields. A better encoding might be to set NumFields + // before pkg. setPkg panics with a nil package, which may be possible + // to reach with invalid packages (and perhaps valid packages, too?), so + // (arbitrarily) set the localpkg if available. + switch { + case pkg != nil: + w.setPkg(pkg, true) + case w.p.shallow: + w.setPkg(w.p.localpkg, true) + default: + panic(internalErrorf("no package to set for empty struct")) + } } w.uint64(uint64(n)) for i := 0; i < n; i++ { From b71392afd89560d5b97d530a5751d3a9e6f0dddc Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Tue, 13 Jun 2023 11:47:20 -0400 Subject: [PATCH 03/26] gopls/internal/lsp/cache: reduce importing in analysis This CL is a substantial reorganization of the analysis driver to ensure that export data is imported at most once per batch of packages that are analyzed, instead of once per import edge. This greatly reduces the amount of allocation and computation done during analysis. In cache/analysis.go, Snapshot.Analyze (which now takes a set of PackageIDs, instead of being called singly in a loop) constructs an ephemeral DAG that mirrors the package graph, and then works in parallel postorder over this graph doing analysis. It uses a single FileSet for the whole batch of packages it creates. The subgraph rooted at each node is effectively a types.Importer for that node, as it represents the mapping from PackagePath to *types.Package. We no longer bother with promises or invalidation. We rely on the fact that the graph is relatively cheap to construct, cache hits are cheap to process, and the whole process only occurs after an idle delay of about a second. Also: - In internal/facts, optimize the fact decoder by using a callback. Previously, it was spending a lot of time traversing the API of all imports of a package to build a PackagePath-to-types.Package mapping. For many packages in terraform-provider-aws this visits over 1M objects (!!). But of course this is trivially computed from the new representation. - In internal/gcimporter, IImportShallow now uses a single callback to get all the types.Package symbols from the client, potentially in parallel (and that's what gopls does). The previous separation of "create" and "populate" has gone away. The analysis driver additionally exploits the getPackages callback to efficiently read the package manifest of an export data file, then abort with an error before proceeding to actually decode the rest of the file. With this change, we can process the internal/provider package of the terraform-provider-aws repo in 20s cold, 4s hot. (Before, it would run out of memory.) $ go test -bench=InitialWorkspaceLoad/hashiform ./gopls/internal/regtest/bench BenchmarkInitialWorkspaceLoad/hashiform-8 1 4014521793 ns/op 349570384 alloc_bytes 439230464 in_use_bytes 668992216 total_alloc_bytes PASS Fixes golang/go#60621 Change-Id: Iadeb02f57eb19dcccb639857053b897a60e0a90e Reviewed-on: https://go-review.googlesource.com/c/tools/+/503195 Reviewed-by: Robert Findley TryBot-Result: Gopher Robot Run-TryBot: Alan Donovan Reviewed-by: Alan Donovan --- gopls/internal/lsp/cache/analysis.go | 791 ++++++++++++---------- gopls/internal/lsp/cache/check.go | 36 +- gopls/internal/lsp/cache/maps.go | 12 - gopls/internal/lsp/cache/session.go | 1 - gopls/internal/lsp/cache/snapshot.go | 19 - gopls/internal/lsp/code_action.go | 4 +- gopls/internal/lsp/diagnostics.go | 52 +- gopls/internal/lsp/source/diagnostics.go | 6 +- gopls/internal/lsp/source/rename.go | 2 +- gopls/internal/lsp/source/view.go | 4 +- gopls/internal/regtest/bench/iwl_test.go | 1 + gopls/internal/regtest/bench/repo_test.go | 9 + internal/event/tag/tag.go | 2 +- internal/facts/facts.go | 48 +- internal/facts/imports.go | 4 + internal/gcimporter/gcimporter_test.go | 3 +- internal/gcimporter/iexport.go | 21 +- internal/gcimporter/iimport.go | 123 ++-- internal/gcimporter/shallow_test.go | 49 +- 19 files changed, 652 insertions(+), 535 deletions(-) diff --git a/gopls/internal/lsp/cache/analysis.go b/gopls/internal/lsp/cache/analysis.go index 04e2651f9e6..5ba84278a0f 100644 --- a/gopls/internal/lsp/cache/analysis.go +++ b/gopls/internal/lsp/cache/analysis.go @@ -23,6 +23,7 @@ import ( "sort" "strings" "sync" + "sync/atomic" "golang.org/x/sync/errgroup" "golang.org/x/tools/go/analysis" @@ -34,7 +35,6 @@ import ( "golang.org/x/tools/internal/event/tag" "golang.org/x/tools/internal/facts" "golang.org/x/tools/internal/gcimporter" - "golang.org/x/tools/internal/memoize" "golang.org/x/tools/internal/typeparams" "golang.org/x/tools/internal/typesinternal" ) @@ -43,48 +43,50 @@ import ( DESIGN - An analysis request is for a set of analyzers and an individual - package ID, notated (a*, p). The result is the set of diagnostics - for that package. It could easily be generalized to a set of - packages, (a*, p*), and perhaps should be, to improve performance - versus calling it in a loop. - - The snapshot holds a cache (persistent.Map) of entries keyed by - (a*, p) pairs ("analysisKey") that have been requested so far. Some - of these entries may be invalidated during snapshot cloning after a - modification event. The cache maps each (a*, p) to a promise of - the analysis result or "analysisSummary". The summary contains the - results of analysis (e.g. diagnostics) as well as the intermediate - results required by the recursion, such as serialized types and - facts. - - The promise represents the result of a call to analyzeImpl, which - type-checks a package and then applies a graph of analyzers to it - in parallel postorder. (These graph edges are "horizontal": within - the same package.) First, analyzeImpl reads the source files of - package p, and obtains (recursively) the results of the "vertical" - dependencies (i.e. analyzers applied to the packages imported by - p). Only the subset of analyzers that use facts need be executed - recursively, but even if this subset is empty, the step is still - necessary because it provides type information. It is possible that - a package may need to be type-checked and analyzed twice, for - different subsets of analyzers, but the overlap is typically - insignificant. - - With the file contents and the results of vertical dependencies, - analyzeImpl is then in a position to produce a key representing the - unit of work (parsing, type-checking, and analysis) that it has to - do. The key is a cryptographic hash of the "recipe" for this step, - including the Metadata, the file contents, the set of analyzers, - and the type and fact information from the vertical dependencies. + An analysis request (Snapshot.Analyze) is for a set of Analyzers and + PackageIDs. The result is the set of diagnostics for those + packages. Each request constructs a transitively closed DAG of + nodes, each representing a package, then works bottom up in + parallel postorder calling runCached to ensure that each node's + analysis summary is up to date. The summary contains the analysis + diagnostics as well as the intermediate results required by the + recursion, such as serialized types and facts. + + The entire DAG is ephemeral. Each node in the DAG records the set + of analyzers to run: the complete set for the root packages, and + the "facty" subset for dependencies. Each package is thus analyzed + at most once. The entire DAG shares a single FileSet for parsing + and importing. + + Each node is processed by runCached. It gets the source file + content hashes for package p, and the summaries of its "vertical" + dependencies (direct imports), and from them it computes a key + representing the unit of work (parsing, type-checking, and + analysis) that it has to do. The key is a cryptographic hash of the + "recipe" for this step, including the Metadata, the file contents, + the set of analyzers, and the type and fact information from the + vertical dependencies. The key is sought in a machine-global persistent file-system based cache. If this gopls process, or another gopls process on the same - machine, has already performed this analysis step, analyzeImpl will + machine, has already performed this analysis step, runCached will make a cache hit and load the serialized summary of the results. If - not, it will have to proceed to type-checking and analysis, and - write a new cache entry. The entry contains serialized types - (export data) and analysis facts. + not, it will have to proceed to run() to parse and type-check the + package and then apply a set of analyzers to it. (The set of + analyzers applied to a single package itself forms a graph of + "actions", and it too is evaluated in parallel postorder; these + dependency edges within the same package are called "horizontal".) + Finally it writes a new cache entry. The entry contains serialized + types (export data) and analysis facts. + + Each node in the DAG acts like a go/types importer mapping, + providing a consistent view of packages and their objects: the + mapping for a node is a superset of its dependencies' mappings. + Every node has an associated *types.Package, initially nil. A + package is populated during run (cache miss) by type-checking its + syntax; but for a cache hit, the package is populated lazily, i.e. + not until it later becomes necessary because it is imported + directly or referenced by export data higher up in the DAG. For types, we use "shallow" export data. Historically, the Go compiler always produced a summary of the types for a given package @@ -103,11 +105,8 @@ import ( "Shallow" export data means that the serialized types describe only a single package. If those types mention types from other packages, the type checker may need to request additional packages beyond - just the direct imports. This means type information for the entire - transitive closure of imports may need to be available just in - case. After a cache hit or a cache miss, the summary is - postprocessed so that it contains the union of export data payloads - of all its direct dependencies. + just the direct imports. Type information for the entire transitive + closure of imports is provided (lazily) by the DAG. For correct dependency analysis, the digest used as a cache key must reflect the "deep" export data, so it is derived recursively @@ -119,8 +118,9 @@ import ( but if its export data is unchanged as a result, then indirect consumers may not need to be re-executed. This allows, for example, one to insert a print statement in a function and not "rebuild" the - whole application (though export data does record line numbers of - types which may be perturbed by otherwise insignificant changes.) + whole application (though export data does record line numbers and + offsets of types which may be perturbed by otherwise insignificant + changes.) The summary must record whether a package is transitively error-free (whether it would compile) because many analyzers are @@ -133,13 +133,6 @@ import ( */ // TODO(adonovan): -// - Profile + optimize: -// - on a cold run, mostly type checking + export data, unsurprisingly. -// - on a hot-disk run, mostly type checking the IWL. -// Would be nice to have a benchmark that separates this out. -// - measure and record in the code the typical operation times -// and file sizes (export data + facts = cache entries). -// - Do "port the old logic" tasks (see TODO in actuallyAnalyze). // - Add a (white-box) test of pruning when a change doesn't affect export data. // - Optimise pruning based on subset of packages mentioned in exportdata. // - Better logging so that it is possible to deduce why an analyzer @@ -147,7 +140,6 @@ import ( // Even if the ultimate consumer decides to ignore errors, // tests and other situations want to be assured of freedom from // errors, not just missing results. This should be recorded. -// - Check that the event trace is intelligible. // - Split this into a subpackage, gopls/internal/lsp/cache/driver, // consisting of this file and three helpers from errors.go. // The (*snapshot).Analyze method would stay behind and make calls @@ -157,11 +149,6 @@ import ( // Metadata(PackageID) source.Metadata // ReadFile(Context, URI) (source.FileHandle, error) // View() *View // for Options -// - define a State type that encapsulates the persistent map -// (with its own mutex), and has methods: -// New() *State -// Clone(invalidate map[PackageID]bool) *State -// Destroy() // - share cache.{goVersionRx,parseGoImpl} // Analyze applies a set of analyzers to the package denoted by id, @@ -171,8 +158,19 @@ import ( // // Precondition: all analyzers within the process have distinct names. // (The names are relied on by the serialization logic.) -func (s *snapshot) Analyze(ctx context.Context, id PackageID, analyzers []*source.Analyzer) ([]*source.Diagnostic, error) { - ctx, done := event.Start(ctx, "snapshot.Analyze", tag.Package.Of(string(id))) +func (snapshot *snapshot) Analyze(ctx context.Context, pkgs map[PackageID]unit, analyzers []*source.Analyzer) ([]*source.Diagnostic, error) { + var tagStr string // sorted comma-separated list of PackageIDs + { + // TODO(adonovan): replace with a generic map[S]any -> string + // function in the tag package, and use maps.Keys + slices.Sort. + keys := make([]string, 0, len(pkgs)) + for id := range pkgs { + keys = append(keys, string(id)) + } + sort.Strings(keys) + tagStr = strings.Join(keys, ",") + } + ctx, done := event.Start(ctx, "snapshot.Analyze", tag.Package.Of(tagStr)) defer done() // Filter and sort enabled root analyzers. @@ -180,7 +178,7 @@ func (s *snapshot) Analyze(ctx context.Context, id PackageID, analyzers []*sourc toSrc := make(map[*analysis.Analyzer]*source.Analyzer) var enabled []*analysis.Analyzer for _, a := range analyzers { - if a.IsEnabled(s.view.Options()) { + if a.IsEnabled(snapshot.view.Options()) { toSrc[a.Analyzer] = a enabled = append(enabled, a.Analyzer) } @@ -188,21 +186,149 @@ func (s *snapshot) Analyze(ctx context.Context, id PackageID, analyzers []*sourc sort.Slice(enabled, func(i, j int) bool { return enabled[i].Name < enabled[j].Name }) + analyzers = nil // prevent accidental use // Register fact types of required analyzers. - for _, a := range requiredAnalyzers(enabled) { - for _, f := range a.FactTypes { - gob.Register(f) + enabled = requiredAnalyzers(enabled) + var useFacts []*analysis.Analyzer + for _, a := range enabled { + if len(a.FactTypes) > 0 { + useFacts = append(useFacts, a) + for _, f := range a.FactTypes { + gob.Register(f) // <2us + } } } + useFacts = requiredAnalyzers(useFacts) - // Run the analysis. - res, err := s.analyze(ctx, id, enabled) - if err != nil { - return nil, err + // File set for this batch (entire graph) of analysis. + fset := token.NewFileSet() + + // Starting from the root packages and following DepsByPkgPath, + // build the DAG of packages we're going to analyze. + // + // Root nodes will run the enabled set of analyzers, + // whereas dependencies will run only the useFacts set. + nodes := make(map[PackageID]*analysisNode) + var leaves []*analysisNode // nodes with no unfinished successors + var makeNode func(from *analysisNode, id PackageID) (*analysisNode, error) + makeNode = func(from *analysisNode, id PackageID) (*analysisNode, error) { + an, ok := nodes[id] + if !ok { + m := snapshot.Metadata(id) + if m == nil { + return nil, bug.Errorf("no metadata for %s", id) + } + + // -- preorder -- + + an = &analysisNode{ + fset: fset, + m: m, + analyzers: useFacts, // all nodes run at least the facty analyzers + allDeps: make(map[PackagePath]*analysisNode), + exportDeps: make(map[PackagePath]*analysisNode), + } + // Unsafe must use a distinguished types.Package. + // Mark it as fully populated from birth. + if m.PkgPath == "unsafe" { + an.typesOnce.Do(func() { an.types = types.Unsafe }) + } + nodes[id] = an + + // -- recursion -- + + // Build subgraphs for dependencies. + an.succs = make(map[PackageID]*analysisNode, len(m.DepsByPkgPath)) + for _, depID := range m.DepsByPkgPath { + dep, err := makeNode(an, depID) + if err != nil { + return nil, err + } + an.succs[depID] = dep + + // Compute the union of all dependencies. + // (This step has quadratic complexity.) + for pkgPath, node := range dep.allDeps { + an.allDeps[pkgPath] = node + } + } + + // -- postorder -- + + an.allDeps[m.PkgPath] = an // add self entry (reflexive transitive closure) + + // Add leaf nodes (no successors) directly to queue. + if len(an.succs) == 0 { + leaves = append(leaves, an) + } + + // Load the contents of each compiled Go file through + // the snapshot's cache. (These are all cache hits as + // files are pre-loaded following packages.Load) + an.files = make([]source.FileHandle, len(m.CompiledGoFiles)) + for i, uri := range m.CompiledGoFiles { + fh, err := snapshot.ReadFile(ctx, uri) + if err != nil { + return nil, err + } + an.files[i] = fh + } + } + // Add edge from predecessor. + if from != nil { + atomic.AddInt32(&from.count, 1) // TODO(adonovan): use generics + an.preds = append(an.preds, from) + } + return an, nil + } + + // For root packages, we run the enabled set of analyzers. + var roots []*analysisNode + for id := range pkgs { + root, err := makeNode(nil, id) + if err != nil { + return nil, err + } + root.analyzers = enabled + roots = append(roots, root) + } + + // Now that we have read all files, + // we no longer need the snapshot. + snapshot = nil + + // Execute phase: run leaves first, adding + // new nodes to the queue as they become leaves. + var g errgroup.Group + // Avoid g.SetLimit here: it makes g.Go stop accepting work, + // which prevents workers from enqeuing, and thus finishing, + // and thus allowing the group to make progress: deadlock. + var enqueue func(it *analysisNode) + enqueue = func(an *analysisNode) { + g.Go(func() error { + summary, err := an.runCached(ctx) + if err != nil { + return err // cancelled, or failed to produce a package + } + an.summary = summary + + // Notify each waiting predecessor, + // and enqueue it when it becomes a leaf. + for _, pred := range an.preds { + if atomic.AddInt32(&pred.count, -1) == 0 { + enqueue(pred) + } + } + return nil + }) + } + for _, leaf := range leaves { + enqueue(leaf) + } + if err := g.Wait(); err != nil { + return nil, err // cancelled, or failed to produce a package } - // Inv: res is the successful result of analyzeImpl(analyzers, id), - // which augments the successful result of actuallyAnalyze. // Report diagnostics only from enabled actions that succeeded. // Errors from creating or analyzing packages are ignored. @@ -215,43 +341,114 @@ func (s *snapshot) Analyze(ctx context.Context, id PackageID, analyzers []*sourc // Even if current callers choose to discard the // results, we should propagate the per-action errors. var results []*source.Diagnostic - for _, a := range enabled { - summary, ok := res.Actions[a.Name] - if summary == nil { - panic(fmt.Sprintf("analyzeSummary.Actions[%q] = (nil, %t); got %v (#60551)", - a.Name, ok, res.Actions)) - } - if summary.Err != "" { - continue // action failed - } - for _, gobDiag := range summary.Diagnostics { - results = append(results, toSourceDiagnostic(toSrc[a], &gobDiag)) + for _, root := range roots { + for _, a := range enabled { + // Inv: root.summary is the successful result of run (via runCached). + summary, ok := root.summary.Actions[a.Name] + if summary == nil { + panic(fmt.Sprintf("analyzeSummary.Actions[%q] = (nil, %t); got %v (#60551)", + a.Name, ok, root.summary.Actions)) + } + if summary.Err != "" { + continue // action failed + } + for _, gobDiag := range summary.Diagnostics { + results = append(results, toSourceDiagnostic(toSrc[a], &gobDiag)) + } } } return results, nil } -// analysisKey is the type of keys in the snapshot.analyses map. -type analysisKey struct { - analyzerNames string - pkgid PackageID +// An analysisNode is a node in a doubly-linked DAG isomorphic to the +// import graph. Each node represents a single package, and the DAG +// represents a batch of analysis work done at once using a single +// realm of token.Pos or types.Object values. +// +// A complete DAG is created anew for each batch of analysis; +// subgraphs are not reused over time. Each node's *types.Package +// field is initially nil and is populated on demand, either from +// type-checking syntax trees (typeCheck) or from importing export +// data (_import). When this occurs, the typesOnce event becomes +// "done". +// +// Each node's allDeps map is a "view" of all its dependencies keyed by +// package path, which defines the types.Importer mapping used when +// populating the node's types.Package. Different nodes have different +// views (e.g. due to variants), but two nodes that are related by +// graph ordering have views that are consistent in their overlap. +// exportDeps is the subset actually referenced by export data; +// this is the set for which we attempt to decode facts. +// +// Each node's run method is called in parallel postorder. On success, +// its summary field is populated, either from the cache (hit), or by +// type-checking and analyzing syntax (miss). +type analysisNode struct { + fset *token.FileSet // file set shared by entire batch (DAG) + m *source.Metadata // metadata for this package + files []source.FileHandle // contents of CompiledGoFiles + analyzers []*analysis.Analyzer // set of analyzers to run + preds []*analysisNode // graph edges: + succs map[PackageID]*analysisNode // (preds -> self -> succs) + allDeps map[PackagePath]*analysisNode // all dependencies including self + exportDeps map[PackagePath]*analysisNode // subset of allDeps ref'd by export data (+self) + count int32 // number of unfinished successors + summary *analyzeSummary // serializable result of analyzing this package + typesOnce sync.Once // guards lazy population of types field + types *types.Package // type information lazily imported from summary } -func (key analysisKey) String() string { - return fmt.Sprintf("%s@%s", key.analyzerNames, key.pkgid) +func (an *analysisNode) String() string { return string(an.m.ID) } + +// _import imports this node's types.Package from export data, if not already done. +// Precondition: analysis was a success. +// Postcondition: an.types and an.exportDeps are populated. +func (an *analysisNode) _import() *types.Package { + an.typesOnce.Do(func() { + an.types = types.NewPackage(string(an.m.PkgPath), string(an.m.Name)) + + // getPackages recursively imports each dependency + // referenced by the export data, in parallel. + getPackages := func(items []gcimporter.GetPackagesItem) error { + var g errgroup.Group + for i, item := range items { + path := PackagePath(item.Path) + dep, ok := an.allDeps[path] + if !ok { + // This early return bypasses Wait; that's ok. + return fmt.Errorf("%s: unknown dependency %q", an.m, path) + } + an.exportDeps[path] = dep // record, for later fact decoding + if dep == an { + items[i].Pkg = an.types + } else { + i := i + g.Go(func() error { + items[i].Pkg = dep._import() + return nil + }) + } + } + return g.Wait() + } + pkg, err := gcimporter.IImportShallow(an.fset, getPackages, an.summary.Export, string(an.m.PkgPath)) + if err != nil { + log.Fatalf("%s: invalid export data: %v", an.m, err) + } + if pkg != an.types { + log.Fatalf("%s: inconsistent packages", an.m) + } + }) + return an.types } // analyzeSummary is a gob-serializable summary of successfully // applying a list of analyzers to a package. type analyzeSummary struct { - PkgPath PackagePath // types.Package.Path() (needed to decode export data) - Export []byte + Export []byte // encoded types of package DeepExportHash source.Hash // hash of reflexive transitive closure of export data Compiles bool // transitively free of list/parse/type errors Actions actionsMap // map from analyzer name to analysis results (*actionSummary) - - // Not serialized: populated after the summary is computed or deserialized. - allExport map[PackagePath][]byte // transitive export data } // actionsMap defines a stable Gob encoding for a map. @@ -302,126 +499,16 @@ type actionSummary struct { Err string // "" => success } -// analyze is a memoization of analyzeImpl. -func (s *snapshot) analyze(ctx context.Context, id PackageID, analyzers []*analysis.Analyzer) (*analyzeSummary, error) { - // Use the caller-sorted list of names of analyzers in the key. - // - // TODO(adonovan): opt: account for analysis results at a - // finer grain to avoid duplicate work when a - // a proper subset of analyzers is requested? - // In particular, TypeErrorAnalyzers don't use facts - // but need to request vdeps just for type information. - names := make([]string, 0, len(analyzers)) - for _, a := range analyzers { - names = append(names, a.Name) - } - // This key describes the result of applying a list of analyzers to a package. - key := analysisKey{strings.Join(names, ","), id} - - // An analysisPromise represents the result of loading, parsing, - // type-checking and analyzing a single package. - type analysisPromise struct { - promise *memoize.Promise // [analyzeImplResult] - } - - type analyzeImplResult struct { - summary *analyzeSummary - err error - } - - // Access the map once, briefly, and atomically. - s.mu.Lock() - entry, hit := s.analyses.Get(key) - if !hit { - entry = analysisPromise{ - promise: memoize.NewPromise("analysis", func(ctx context.Context, arg interface{}) interface{} { - summary, err := analyzeImpl(ctx, arg.(*snapshot), analyzers, id) - return analyzeImplResult{summary, err} - }), - } - s.analyses.Set(key, entry, nil) // nothing needs releasing - } - s.mu.Unlock() - - // Await result. - ap := entry.(analysisPromise) - v, err := s.awaitPromise(ctx, ap.promise) - if err != nil { - return nil, err // e.g. cancelled - } - res := v.(analyzeImplResult) - return res.summary, res.err -} - -// analyzeImpl applies a list of analyzers (plus any others +// runCached applies a list of analyzers (plus any others // transitively required by them) to a package. It succeeds as long // as it could produce a types.Package, even if there were direct or // indirect list/parse/type errors, and even if all the analysis // actions failed. It usually fails only if the package was unknown, // a file was missing, or the operation was cancelled. // -// Postcondition: analyzeImpl must not continue to use the snapshot +// Postcondition: runCached must not continue to use the snapshot // (in background goroutines) after it has returned; see memoize.RefCounted. -func analyzeImpl(ctx context.Context, snapshot *snapshot, analyzers []*analysis.Analyzer, id PackageID) (*analyzeSummary, error) { - m := snapshot.Metadata(id) - if m == nil { - return nil, fmt.Errorf("no metadata for %s", id) - } - - // Also, load the contents of each "compiled" Go file through - // the snapshot's cache. - // (These are all cache hits as files are pre-loaded following packages.Load) - compiledGoFiles := make([]source.FileHandle, len(m.CompiledGoFiles)) - for i, uri := range m.CompiledGoFiles { - fh, err := snapshot.ReadFile(ctx, uri) - if err != nil { - return nil, err // e.g. canceled - } - compiledGoFiles[i] = fh - } - - // Recursively analyze each "vertical" dependency - // for its types.Package and (perhaps) analysis.Facts. - // If any of them fails to produce a package, we cannot continue. - // We request only the analyzers that produce facts. - vdeps := make(map[PackageID]*analyzeSummary) - { - var group errgroup.Group - - // Analyze vertical dependencies. - // We request only the required analyzers that use facts. - var useFacts []*analysis.Analyzer - for _, a := range requiredAnalyzers(analyzers) { - if len(a.FactTypes) > 0 { - useFacts = append(useFacts, a) - } - } - var vdepsMu sync.Mutex - for _, id := range m.DepsByPkgPath { - id := id - group.Go(func() error { - res, err := snapshot.analyze(ctx, id, useFacts) - if err != nil { - return err // cancelled, or failed to produce a package - } - - vdepsMu.Lock() - vdeps[id] = res - vdepsMu.Unlock() - return nil - }) - } - - if err := group.Wait(); err != nil { - return nil, err - } - } - - // Inv: analyze() of all vdeps succeeded (though some actions may have failed). - - // We no longer depend on the snapshot. - snapshot = nil - +func (an *analysisNode) runCached(ctx context.Context) (*analyzeSummary, error) { // At this point we have the action results (serialized // packages and facts) of our immediate dependencies, // and the metadata and content of this package. @@ -433,7 +520,7 @@ func analyzeImpl(ctx context.Context, snapshot *snapshot, analyzers []*analysis. // The hash of our inputs is based on the serialized export // data and facts so that immaterial changes can be pruned // without decoding. - key := analysisCacheKey(analyzers, m, compiledGoFiles, vdeps) + key := an.cacheKey() // Access the cache. var summary *analyzeSummary @@ -446,14 +533,14 @@ func analyzeImpl(ctx context.Context, snapshot *snapshot, analyzers []*analysis. } else { // Cache miss: do the work. var err error - summary, err = actuallyAnalyze(ctx, analyzers, m, vdeps, compiledGoFiles) + summary, err = an.run(ctx) if err != nil { return nil, err } go func() { data := mustEncode(summary) if false { - log.Printf("Set key=%d value=%d id=%s\n", len(key), len(data), id) + log.Printf("Set key=%d value=%d id=%s\n", len(key), len(data), an.m.ID) } if err := filecache.Set(cacheKind, key, data); err != nil { event.Error(ctx, "internal error updating analysis shared cache", err) @@ -461,34 +548,15 @@ func analyzeImpl(ctx context.Context, snapshot *snapshot, analyzers []*analysis. }() } - // Hit or miss, we need to merge the export data from - // dependencies so that it includes all the types - // that might be summoned by the type checker. - // - // TODO(adonovan): opt: reduce this set by recording - // which packages were actually summoned by insert(). - // (Just makes map smaller; probably marginal?) - allExport := make(map[PackagePath][]byte) - for _, vdep := range vdeps { - for k, v := range vdep.allExport { - allExport[k] = v - } - } - allExport[m.PkgPath] = summary.Export - summary.allExport = allExport - return summary, nil } // analysisCacheKey returns a cache key that is a cryptographic digest // of the all the values that might affect type checking and analysis: // the analyzer names, package metadata, names and contents of -// compiled Go files, and vdeps information (export data and facts). -// -// TODO(adonovan): safety: define our own flavor of Metadata -// containing just the fields we need, and using it in the subsequent -// logic, to keep us honest about hashing all parts that matter? -func analysisCacheKey(analyzers []*analysis.Analyzer, m *source.Metadata, compiledGoFiles []source.FileHandle, vdeps map[PackageID]*analyzeSummary) [sha256.Size]byte { +// compiled Go files, and vdeps (successor) information +// (export data and facts). +func (an *analysisNode) cacheKey() [sha256.Size]byte { hasher := sha256.New() // In principle, a key must be the hash of an @@ -496,12 +564,13 @@ func analysisCacheKey(analyzers []*analysis.Analyzer, m *source.Metadata, compil // If it's ambiguous, we risk collisions. // analyzers - fmt.Fprintf(hasher, "analyzers: %d\n", len(analyzers)) - for _, a := range analyzers { + fmt.Fprintf(hasher, "analyzers: %d\n", len(an.analyzers)) + for _, a := range an.analyzers { fmt.Fprintln(hasher, a.Name) } // package metadata + m := an.m fmt.Fprintf(hasher, "package: %s %s %s\n", m.ID, m.Name, m.PkgPath) // We can ignore m.DepsBy{Pkg,Import}Path: although the logic // uses those fields, we account for them by hashing vdeps. @@ -521,30 +590,31 @@ func analysisCacheKey(analyzers []*analysis.Analyzer, m *source.Metadata, compil } // file names and contents - fmt.Fprintf(hasher, "files: %d\n", len(compiledGoFiles)) - for _, fh := range compiledGoFiles { + fmt.Fprintf(hasher, "files: %d\n", len(an.files)) + for _, fh := range an.files { fmt.Fprintln(hasher, fh.FileIdentity()) } // vdeps, in PackageID order - depIDs := make([]string, 0, len(vdeps)) - for depID := range vdeps { + depIDs := make([]string, 0, len(an.succs)) + for depID := range an.succs { depIDs = append(depIDs, string(depID)) } - sort.Strings(depIDs) + sort.Strings(depIDs) // TODO(adonovan): avoid conversions by using slices.Sort[PackageID] for _, depID := range depIDs { - vdep := vdeps[PackageID(depID)] - fmt.Fprintf(hasher, "dep: %s\n", vdep.PkgPath) - fmt.Fprintf(hasher, "export: %s\n", vdep.DeepExportHash) + vdep := an.succs[PackageID(depID)] + fmt.Fprintf(hasher, "dep: %s\n", vdep.m.PkgPath) + fmt.Fprintf(hasher, "export: %s\n", vdep.summary.DeepExportHash) // action results: errors and facts - names := make([]string, 0, len(vdep.Actions)) - for name := range vdep.Actions { + actions := vdep.summary.Actions + names := make([]string, 0, len(actions)) + for name := range actions { names = append(names, name) } sort.Strings(names) for _, name := range names { - summary := vdep.Actions[name] + summary := actions[name] fmt.Fprintf(hasher, "action %s\n", name) if summary.Err != "" { fmt.Fprintf(hasher, "error %s\n", summary.Err) @@ -561,28 +631,25 @@ func analysisCacheKey(analyzers []*analysis.Analyzer, m *source.Metadata, compil return hash } -// actuallyAnalyze implements the cache-miss case. +// run implements the cache-miss case. // This function does not access the snapshot. // // Postcondition: on success, the analyzeSummary.Actions // key set is {a.Name for a in analyzers}. -func actuallyAnalyze(ctx context.Context, analyzers []*analysis.Analyzer, m *source.Metadata, vdeps map[PackageID]*analyzeSummary, compiledGoFiles []source.FileHandle) (*analyzeSummary, error) { - // Create a local FileSet for processing this package only. - fset := token.NewFileSet() - +func (an *analysisNode) run(ctx context.Context) (*analyzeSummary, error) { // Parse only the "compiled" Go files. // Do the computation in parallel. - parsed := make([]*source.ParsedGoFile, len(compiledGoFiles)) + parsed := make([]*source.ParsedGoFile, len(an.files)) { var group errgroup.Group - for i, fh := range compiledGoFiles { + for i, fh := range an.files { i, fh := i, fh group.Go(func() error { // Call parseGoImpl directly, not the caching wrapper, // as cached ASTs require the global FileSet. // ast.Object resolution is unfortunately an implied part of the // go/analysis contract. - pgf, err := parseGoImpl(ctx, fset, fh, source.ParseFull&^source.SkipObjectResolution) + pgf, err := parseGoImpl(ctx, an.fset, fh, source.ParseFull&^source.SkipObjectResolution) parsed[i] = pgf return err }) @@ -592,22 +659,43 @@ func actuallyAnalyze(ctx context.Context, analyzers []*analysis.Analyzer, m *sou } } - // Type-check the package. - pkg := typeCheckForAnalysis(fset, parsed, m, vdeps) + // Type-check the package syntax. + pkg := an.typeCheck(parsed) + + // Compute the union of exportDeps across our direct imports. + // This is the set that will be needed by the fact decoder. + allExportDeps := make(map[PackagePath]*analysisNode) + for _, succ := range an.succs { + for k, v := range succ.exportDeps { + allExportDeps[k] = v + } + } + + // The fact decoder needs a means to look up a Package by path. + pkg.factsDecoder = facts.NewDecoderFunc(pkg.types, func(path string) *types.Package { + // Note: Decode is called concurrently, and thus so is this function. + + // Does the fact relate to a package referenced by export data? + if dep, ok := allExportDeps[PackagePath(path)]; ok { + dep.typesOnce.Do(func() { log.Fatal("dep.types not populated") }) + return dep.types + } - // Build a map of PkgPath to *Package for all packages mentioned - // in exportdata for use by facts. - pkg.factsDecoder = facts.NewDecoder(pkg.types) + // If the fact relates to a dependency not referenced + // by export data, it is safe to ignore it. + // (In that case dep.types exists but may be unpopulated + // or in the process of being populated from export data.) + if an.allDeps[PackagePath(path)] == nil { + log.Fatalf("fact package %q is not a dependency", path) + } + return nil + }) // Poll cancellation state. if err := ctx.Err(); err != nil { return nil, err } - // TODO(adonovan): port the old logic to: - // - gather go/packages diagnostics from m.Errors? (port goPackagesErrorDiagnostics) - // - gather diagnostics from expandErrors + typeErrorDiagnostics + depsErrors. - // -- analysis -- // Build action graph for this package. @@ -621,7 +709,7 @@ func actuallyAnalyze(ctx context.Context, analyzers []*analysis.Analyzer, m *sou for _, req := range a.Requires { hdeps = append(hdeps, mkAction(req)) } - act = &action{a: a, pkg: pkg, vdeps: vdeps, hdeps: hdeps} + act = &action{a: a, pkg: pkg, vdeps: an.succs, hdeps: hdeps} actions[a] = act } return act @@ -629,7 +717,7 @@ func actuallyAnalyze(ctx context.Context, analyzers []*analysis.Analyzer, m *sou // Build actions for initial package. var roots []*action - for _, a := range analyzers { + for _, a := range an.analyzers { roots = append(roots, mkAction(a)) } @@ -652,7 +740,6 @@ func actuallyAnalyze(ctx context.Context, analyzers []*analysis.Analyzer, m *sou } return &analyzeSummary{ - PkgPath: PackagePath(pkg.types.Path()), Export: pkg.export, DeepExportHash: pkg.deepExportHash, Compiles: pkg.compiles, @@ -660,9 +747,13 @@ func actuallyAnalyze(ctx context.Context, analyzers []*analysis.Analyzer, m *sou }, nil } -func typeCheckForAnalysis(fset *token.FileSet, parsed []*source.ParsedGoFile, m *source.Metadata, vdeps map[PackageID]*analyzeSummary) *analysisPackage { +// Postcondition: an.types and an.exportDeps are populated. +func (an *analysisNode) typeCheck(parsed []*source.ParsedGoFile) *analysisPackage { + m := an.m + fset := an.fset + if false { // debugging - log.Println("typeCheckForAnalysis", m.PkgPath) + log.Println("typeCheck", m.ID) } pkg := &analysisPackage{ @@ -684,6 +775,11 @@ func typeCheckForAnalysis(fset *token.FileSet, parsed []*source.ParsedGoFile, m } typeparams.InitInstanceInfo(pkg.typesInfo) + // Unsafe has no syntax. + if m.PkgPath == "unsafe" { + return pkg + } + for i, p := range parsed { pkg.files[i] = p.File if p.ParseErr != nil { @@ -691,77 +787,12 @@ func typeCheckForAnalysis(fset *token.FileSet, parsed []*source.ParsedGoFile, m } } - // Unsafe is special. - if m.PkgPath == "unsafe" { - pkg.types = types.Unsafe - return pkg - } - - // Compute the union of transitive export data. - // (The actual values are shared, and not serialized.) - allExport := make(map[PackagePath][]byte) - for _, vdep := range vdeps { - for k, v := range vdep.allExport { - allExport[k] = v - } - - if !vdep.Compiles { + for _, vdep := range an.succs { + if !vdep.summary.Compiles { pkg.compiles = false // transitive error } } - // exportHasher computes a hash of the names and export data of - // each package that was actually loaded during type checking. - // - // Because we use shallow export data, the hash for dependency - // analysis must incorporate indirect dependencies. As an - // optimization, we include only those that were actually - // used, which may be a small subset of those available. - // - // TODO(adonovan): opt: even better would be to implement a - // traversal over the package API like facts.NewDecoder does - // and only mention that set of packages in the hash. - // Perhaps there's a way to do that more efficiently. - // - // TODO(adonovan): opt: record the shallow hash alongside the - // shallow export data in the allExport map to avoid repeatedly - // hashing the export data. - // - // The writes to hasher below assume that type checking imports - // packages in a deterministic order. - exportHasher := sha256.New() - hashExport := func(pkgPath PackagePath, export []byte) { - fmt.Fprintf(exportHasher, "%s %d ", pkgPath, len(export)) - exportHasher.Write(export) - } - - // importer state - var ( - insert func(p *types.Package, name string) - importMap = make(map[string]*types.Package) // keys are PackagePaths - ) - loadFromExportData := func(pkgPath PackagePath) (*types.Package, error) { - export, ok := allExport[pkgPath] - if !ok { - return nil, bug.Errorf("missing export data for %q", pkgPath) - } - hashExport(pkgPath, export) - imported, err := gcimporter.IImportShallow(fset, gcimporter.GetPackageFromMap(importMap), export, string(pkgPath), insert) - if err != nil { - return nil, bug.Errorf("invalid export data for %q: %v", pkgPath, err) - } - return imported, nil - } - insert = func(p *types.Package, name string) { - imported, err := loadFromExportData(PackagePath(p.Path())) - if err != nil { - log.Fatalf("internal error: %v", err) - } - if imported != p { - log.Fatalf("internal error: inconsistent packages") - } - } - cfg := &types.Config{ Sizes: m.TypesSizes, Error: func(e error) { @@ -778,16 +809,13 @@ func typeCheckForAnalysis(fset *token.FileSet, parsed []*source.ParsedGoFile, m pkg.typeErrors = append(pkg.typeErrors, typeError) }, Importer: importerFunc(func(importPath string) (*types.Package, error) { - if importPath == "unsafe" { - return types.Unsafe, nil // unsafe has no export data - } - // Beware that returning an error from this function // will cause the type checker to synthesize a fake // package whose Path is importPath, potentially // losing a vendor/ prefix. If type-checking errors // are swallowed, these packages may be confusing. + // Map ImportPath to ID. id, ok := m.DepsByImpPath[ImportPath(importPath)] if !ok { // The import syntax is inconsistent with the metadata. @@ -798,22 +826,23 @@ func typeCheckForAnalysis(fset *token.FileSet, parsed []*source.ParsedGoFile, m return nil, fmt.Errorf("missing metadata for import of %q", importPath) } - depResult, ok := vdeps[id] // id may be "" - if !ok { + // Map ID to node. (id may be "") + dep := an.succs[id] + if dep == nil { // Analogous to (*snapshot).missingPkgError // in the logic for regular type-checking, // but without a snapshot we can't provide // such detail, and anyway most analysis // failures aren't surfaced in the UI. - return nil, fmt.Errorf("no required module provides package %q (id=%q)", importPath, id) + return nil, fmt.Errorf("no required module provides analysis package %q (id=%q)", importPath, id) } // (Duplicates logic from check.go.) - if !source.IsValidImport(m.PkgPath, depResult.PkgPath) { + if !source.IsValidImport(an.m.PkgPath, dep.m.PkgPath) { return nil, fmt.Errorf("invalid use of internal package %s", importPath) } - return loadFromExportData(depResult.PkgPath) + return dep._import(), nil }), } @@ -845,7 +874,13 @@ func typeCheckForAnalysis(fset *token.FileSet, parsed []*source.ParsedGoFile, m } } - // Emit the export data and compute the deep hash. + // Publish the completed package. + an.typesOnce.Do(func() { an.types = pkg.types }) + if an.types != pkg.types { + log.Fatalf("typesOnce prematurely done") + } + + // Emit the export data and compute the recursive hash. export, err := gcimporter.IExportShallow(pkg.fset, pkg.types) if err != nil { // TODO(adonovan): in light of exporter bugs such as #57729, @@ -854,12 +889,56 @@ func typeCheckForAnalysis(fset *token.FileSet, parsed []*source.ParsedGoFile, m log.Fatalf("internal error writing shallow export data: %v", err) } pkg.export = export - hashExport(m.PkgPath, export) - exportHasher.Sum(pkg.deepExportHash[:0]) + + // Compute a recursive hash to account for the export data of + // this package and each dependency referenced by it. + // Also, populate exportDeps. + hash := sha256.New() + fmt.Fprintf(hash, "%s %d\n", m.PkgPath, len(export)) + hash.Write(export) + paths, err := readShallowManifest(export) + if err != nil { + log.Fatalf("internal error: bad export data: %v", err) + } + for _, path := range paths { + dep, ok := an.allDeps[PackagePath(path)] + if !ok { + log.Fatalf("%s: missing dependency: %q", an, path) + } + fmt.Fprintf(hash, "%s %s\n", dep.m.PkgPath, dep.summary.DeepExportHash) + an.exportDeps[PackagePath(path)] = dep + } + an.exportDeps[m.PkgPath] = an // self + hash.Sum(pkg.deepExportHash[:0]) return pkg } +// readShallowManifest returns the manifest of packages referenced by +// a shallow export data file for a package (excluding the package itself). +// TODO(adonovan): add a test. +func readShallowManifest(export []byte) ([]PackagePath, error) { + const selfPath = "" // dummy path + var paths []PackagePath + getPackages := func(items []gcimporter.GetPackagesItem) error { + paths = []PackagePath{} // non-nil + for _, item := range items { + if item.Path != selfPath { + paths = append(paths, PackagePath(item.Path)) + } + } + return errors.New("stop") // terminate importer + } + _, err := gcimporter.IImportShallow(token.NewFileSet(), getPackages, export, selfPath) + if paths == nil { + if err != nil { + return nil, err // failed before getPackages callback + } + return nil, bug.Errorf("internal error: IImportShallow did not call getPackages") + } + return paths, nil // success +} + // analysisPackage contains information about a package, including // syntax trees, used transiently during its type-checking and analysis. type analysisPackage struct { @@ -885,8 +964,8 @@ type action struct { once sync.Once a *analysis.Analyzer pkg *analysisPackage - hdeps []*action // horizontal dependencies - vdeps map[PackageID]*analyzeSummary // vertical dependencies + hdeps []*action // horizontal dependencies + vdeps map[PackageID]*analysisNode // vertical dependencies // results of action.exec(): result interface{} // result of Run function, of type a.ResultType @@ -944,9 +1023,9 @@ func (act *action) exec() (interface{}, *actionSummary, error) { // we return the dependencies' error' unadorned. if hasFacts { // TODO(adonovan): use deterministic order. - for _, res := range act.vdeps { - if vdep := res.Actions[analyzer.Name]; vdep.Err != "" { - return nil, nil, errors.New(vdep.Err) + for _, vdep := range act.vdeps { + if summ := vdep.summary.Actions[analyzer.Name]; summ.Err != "" { + return nil, nil, errors.New(summ.Err) } } } @@ -977,15 +1056,14 @@ func (act *action) exec() (interface{}, *actionSummary, error) { // efficient to fork and tailor it to our precise needs. // // We've already sharded the fact encoding by action - // so that it can be done in parallel (hoisting the - // ImportMap call so that we build the map once per package). + // so that it can be done in parallel. // We could eliminate locking. // We could also dovetail more closely with the export data // decoder to obtain a more compact representation of // packages and objects (e.g. its internal IDs, instead // of PkgPaths and objectpaths.) - // Read and decode analysis facts for each imported package. + // Read and decode analysis facts for each direct import. factset, err := pkg.factsDecoder.Decode(func(imp *types.Package) ([]byte, error) { if !hasFacts { return nil, nil // analyzer doesn't use facts, so no vdeps @@ -1012,11 +1090,12 @@ func (act *action) exec() (interface{}, *actionSummary, error) { return nil, nil } - vdep, ok := act.vdeps[id] - if !ok { + vdep := act.vdeps[id] + if vdep == nil { return nil, bug.Errorf("internal error in %s: missing vdep for id=%s", pkg.types.Path(), id) } - return vdep.Actions[analyzer.Name].Facts, nil + + return vdep.summary.Actions[analyzer.Name].Facts, nil }) if err != nil { return nil, nil, fmt.Errorf("internal error decoding analysis facts: %w", err) diff --git a/gopls/internal/lsp/cache/check.go b/gopls/internal/lsp/cache/check.go index 59b19e03bab..81c81a30b5e 100644 --- a/gopls/internal/lsp/cache/check.go +++ b/gopls/internal/lsp/cache/check.go @@ -531,29 +531,19 @@ func (b *typeCheckBatch) importPackage(ctx context.Context, m *source.Metadata, impMap := b.importMap(m.ID) - var firstErr error // TODO(rfindley): unused: revisit or remove. thisPackage := types.NewPackage(string(m.PkgPath), string(m.Name)) - getPackage := func(path, name string) *types.Package { - if path == string(m.PkgPath) { - return thisPackage - } - - id := impMap[path] - imp, err := b.getImportPackage(ctx, id) - if err == nil { - return imp - } - // inv: err != nil - if firstErr == nil { - firstErr = err + getPackages := func(items []gcimporter.GetPackagesItem) error { + for i, item := range items { + if item.Path == string(m.PkgPath) { + items[i].Pkg = thisPackage + } else { + pkg, err := b.getImportPackage(ctx, impMap[item.Path]) + if err != nil { + return err + } + items[i].Pkg = pkg + } } - - // Context cancellation, or a very bad error such as a file permission - // error. - // - // Returning nil here will cause the import to fail (and panic if - // gcimporter.debug is set), but that is preferable to the confusing errors - // produced when shallow import encounters an empty package. return nil } @@ -563,9 +553,9 @@ func (b *typeCheckBatch) importPackage(ctx context.Context, m *source.Metadata, return nil, ctx.Err() } - // TODO(rfindley): collect "deep" hashes here using the provided + // TODO(rfindley): collect "deep" hashes here using the getPackages // callback, for precise pruning. - imported, err := gcimporter.IImportShallow(b.fset, getPackage, data, string(m.PkgPath), func(*types.Package, string) {}) + imported, err := gcimporter.IImportShallow(b.fset, getPackages, data, string(m.PkgPath)) if err != nil { return nil, fmt.Errorf("import failed for %q: %v", m.ID, err) } diff --git a/gopls/internal/lsp/cache/maps.go b/gopls/internal/lsp/cache/maps.go index 533c3397b42..de6187da255 100644 --- a/gopls/internal/lsp/cache/maps.go +++ b/gopls/internal/lsp/cache/maps.go @@ -5,8 +5,6 @@ package cache import ( - "strings" - "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/gopls/internal/span" "golang.org/x/tools/internal/persistent" @@ -136,13 +134,3 @@ func (s knownDirsSet) Insert(key span.URI) { func (s knownDirsSet) Remove(key span.URI) { s.impl.Delete(key) } - -// analysisKeyLessInterface is the less-than relation for analysisKey -// values wrapped in an interface. -func analysisKeyLessInterface(a, b interface{}) bool { - x, y := a.(analysisKey), b.(analysisKey) - if cmp := strings.Compare(x.analyzerNames, y.analyzerNames); cmp != 0 { - return cmp < 0 - } - return x.pkgid < y.pkgid -} diff --git a/gopls/internal/lsp/cache/session.go b/gopls/internal/lsp/cache/session.go index 8eae64e4d82..fd2e4d2f207 100644 --- a/gopls/internal/lsp/cache/session.go +++ b/gopls/internal/lsp/cache/session.go @@ -171,7 +171,6 @@ func (s *Session) createView(ctx context.Context, name string, folder span.URI, parseCache: new(parseCache), activePackages: persistent.NewMap(packageIDLessInterface), symbolizeHandles: persistent.NewMap(uriLessInterface), - analyses: persistent.NewMap(analysisKeyLessInterface), workspacePackages: make(map[PackageID]PackagePath), unloadableFiles: make(map[span.URI]struct{}), parseModHandles: persistent.NewMap(uriLessInterface), diff --git a/gopls/internal/lsp/cache/snapshot.go b/gopls/internal/lsp/cache/snapshot.go index 6ff14fb6d34..ce98fe6d02f 100644 --- a/gopls/internal/lsp/cache/snapshot.go +++ b/gopls/internal/lsp/cache/snapshot.go @@ -122,11 +122,6 @@ type snapshot struct { // IDs not contained in the map are not known to be open or not open. activePackages *persistent.Map // from packageID to *Package - // analyses maps an analysisKey (which identifies a package - // and a set of analyzers) to the handle for the future result - // of loading the package and analyzing it. - analyses *persistent.Map // from analysisKey to analysisPromise - // workspacePackages contains the workspace's packages, which are loaded // when the view is created. It contains no intermediate test variants. workspacePackages map[PackageID]PackagePath @@ -268,7 +263,6 @@ func (s *snapshot) destroy(destroyedBy string) { s.packages.Destroy() s.activePackages.Destroy() - s.analyses.Destroy() s.files.Destroy() s.knownSubdirs.Destroy() s.symbolizeHandles.Destroy() @@ -2014,7 +2008,6 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC initializedErr: s.initializedErr, packages: s.packages.Clone(), activePackages: s.activePackages.Clone(), - analyses: s.analyses.Clone(), files: s.files.Clone(), parseCache: s.parseCache, symbolizeHandles: s.symbolizeHandles.Clone(), @@ -2242,18 +2235,6 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC result.activePackages.Delete(id) } - // Delete invalidated analysis actions. - var actionsToDelete []analysisKey - result.analyses.Range(func(k, _ interface{}) { - key := k.(analysisKey) - if _, ok := idsToInvalidate[key.pkgid]; ok { - actionsToDelete = append(actionsToDelete, key) - } - }) - for _, key := range actionsToDelete { - result.analyses.Delete(key) - } - // If a file has been deleted, we must delete metadata for all packages // containing that file. // diff --git a/gopls/internal/lsp/code_action.go b/gopls/internal/lsp/code_action.go index 8e817b88bb6..39addc8e98b 100644 --- a/gopls/internal/lsp/code_action.go +++ b/gopls/internal/lsp/code_action.go @@ -203,7 +203,7 @@ func (s *Server) codeAction(ctx context.Context, params *protocol.CodeActionPara if err != nil { return nil, err } - analysisDiags, err := source.Analyze(ctx, snapshot, pkg.Metadata().ID, true) + analysisDiags, err := source.Analyze(ctx, snapshot, map[source.PackageID]unit{pkg.Metadata().ID: {}}, true) if err != nil { return nil, err } @@ -575,3 +575,5 @@ func goTest(ctx context.Context, snapshot source.Snapshot, uri span.URI, rng pro Command: &cmd, }}, nil } + +type unit = struct{} diff --git a/gopls/internal/lsp/diagnostics.go b/gopls/internal/lsp/diagnostics.go index 88008d319bc..94b5621b185 100644 --- a/gopls/internal/lsp/diagnostics.go +++ b/gopls/internal/lsp/diagnostics.go @@ -11,6 +11,7 @@ import ( "fmt" "os" "path/filepath" + "sort" "strings" "sync" "time" @@ -362,7 +363,7 @@ func (s *Server) diagnose(ctx context.Context, snapshot source.Snapshot, analyze var ( seen = map[span.URI]struct{}{} toDiagnose = make(map[source.PackageID]*source.Metadata) - toAnalyze = make(map[source.PackageID]*source.Metadata) + toAnalyze = make(map[source.PackageID]unit) ) for _, m := range workspace { var hasNonIgnored, hasOpenFile bool @@ -378,7 +379,7 @@ func (s *Server) diagnose(ctx context.Context, snapshot source.Snapshot, analyze if hasNonIgnored { toDiagnose[m.ID] = m if analyze == analyzeEverything || analyze == analyzeOpenPackages && hasOpenFile { - toAnalyze[m.ID] = m + toAnalyze[m.ID] = unit{} } } } @@ -416,7 +417,7 @@ func (s *Server) diagnose(ctx context.Context, snapshot source.Snapshot, analyze // of concurrent dispatch: as of writing we concurrently run TidyDiagnostics // and diagnosePkgs, and diagnosePkgs concurrently runs PackageDiagnostics and // analysis. -func (s *Server) diagnosePkgs(ctx context.Context, snapshot source.Snapshot, toDiagnose, toAnalyze map[source.PackageID]*source.Metadata) { +func (s *Server) diagnosePkgs(ctx context.Context, snapshot source.Snapshot, toDiagnose map[source.PackageID]*source.Metadata, toAnalyze map[source.PackageID]unit) { ctx, done := event.Start(ctx, "Server.diagnosePkgs", source.SnapshotLabels(snapshot)...) defer done() @@ -425,7 +426,6 @@ func (s *Server) diagnosePkgs(ctx context.Context, snapshot source.Snapshot, toD var ( wg sync.WaitGroup pkgDiags map[span.URI][]*source.Diagnostic - analysisMu sync.Mutex analysisDiags = make(map[span.URI][]*source.Diagnostic) ) @@ -446,28 +446,30 @@ func (s *Server) diagnosePkgs(ctx context.Context, snapshot source.Snapshot, toD // Get diagnostics from analysis framework. // This includes type-error analyzers, which suggest fixes to compiler errors. - // - // TODO(adonovan): in many cases we will be analyze multiple open variants of - // an open package, which have significantly overlapping import graphs. - // It may make sense to change the Analyze API to accept a slice of IDs, or - // merge analysis with type-checking. - for _, m := range toAnalyze { - m := m - wg.Add(1) - go func() { - defer wg.Done() - diags, err := source.Analyze(ctx, snapshot, m.ID, false) - if err != nil { - event.Error(ctx, "warning: analyzing package", err, append(source.SnapshotLabels(snapshot), tag.Package.Of(string(m.ID)))...) - return - } - analysisMu.Lock() - for uri, diags := range diags { - analysisDiags[uri] = append(analysisDiags[uri], diags...) + wg.Add(1) + go func() { + defer wg.Done() + diags, err := source.Analyze(ctx, snapshot, toAnalyze, false) + if err != nil { + var tagStr string // sorted comma-separated list of package IDs + { + // TODO(adonovan): replace with a generic map[S]any -> string + // function in the tag package, and use maps.Keys + slices.Sort. + keys := make([]string, 0, len(toDiagnose)) + for id := range toDiagnose { + keys = append(keys, string(id)) + } + sort.Strings(keys) + tagStr = strings.Join(keys, ",") } - analysisMu.Unlock() - }() - } + event.Error(ctx, "warning: analyzing package", err, append(source.SnapshotLabels(snapshot), tag.Package.Of(tagStr))...) + return + } + for uri, diags := range diags { + analysisDiags[uri] = append(analysisDiags[uri], diags...) + } + }() + wg.Wait() // TODO(rfindley): remove the guards against snapshot.IsBuiltin, after the diff --git a/gopls/internal/lsp/source/diagnostics.go b/gopls/internal/lsp/source/diagnostics.go index 336e35b25ac..2a397dde91f 100644 --- a/gopls/internal/lsp/source/diagnostics.go +++ b/gopls/internal/lsp/source/diagnostics.go @@ -21,7 +21,7 @@ type SuggestedFix struct { } // Analyze reports go/analysis-framework diagnostics in the specified package. -func Analyze(ctx context.Context, snapshot Snapshot, pkgid PackageID, includeConvenience bool) (map[span.URI][]*Diagnostic, error) { +func Analyze(ctx context.Context, snapshot Snapshot, pkgIDs map[PackageID]unit, includeConvenience bool) (map[span.URI][]*Diagnostic, error) { // Exit early if the context has been canceled. This also protects us // from a race on Options, see golang/go#36699. if ctx.Err() != nil { @@ -45,7 +45,7 @@ func Analyze(ctx context.Context, snapshot Snapshot, pkgid PackageID, includeCon } } - analysisDiagnostics, err := snapshot.Analyze(ctx, pkgid, analyzers) + analysisDiagnostics, err := snapshot.Analyze(ctx, pkgIDs, analyzers) if err != nil { return nil, err } @@ -81,7 +81,7 @@ func FileDiagnostics(ctx context.Context, snapshot Snapshot, uri span.URI) (File if err != nil { return nil, nil, err } - adiags, err := Analyze(ctx, snapshot, pkg.Metadata().ID, false) + adiags, err := Analyze(ctx, snapshot, map[PackageID]unit{pkg.Metadata().ID: {}}, false) if err != nil { return nil, nil, err } diff --git a/gopls/internal/lsp/source/rename.go b/gopls/internal/lsp/source/rename.go index 60fb48d6a37..b460cb00121 100644 --- a/gopls/internal/lsp/source/rename.go +++ b/gopls/internal/lsp/source/rename.go @@ -21,7 +21,7 @@ package source // - special cases: embedded fields, interfaces, test variants, // function-local things with uppercase names; // packages with type errors (currently 'satisfy' rejects them), -// pakage with missing imports; +// package with missing imports; // // - measure performance in k8s. // diff --git a/gopls/internal/lsp/source/view.go b/gopls/internal/lsp/source/view.go index 2f7a3f27ef2..ed204f3d85a 100644 --- a/gopls/internal/lsp/source/view.go +++ b/gopls/internal/lsp/source/view.go @@ -92,8 +92,8 @@ type Snapshot interface { // Position information is added to FileSet(). ParseGo(ctx context.Context, fh FileHandle, mode parser.Mode) (*ParsedGoFile, error) - // Analyze runs the specified analyzers on the given package at this snapshot. - Analyze(ctx context.Context, id PackageID, analyzers []*Analyzer) ([]*Diagnostic, error) + // Analyze runs the specified analyzers on the given packages at this snapshot. + Analyze(ctx context.Context, pkgIDs map[PackageID]unit, analyzers []*Analyzer) ([]*Diagnostic, error) // RunGoCommandPiped runs the given `go` command, writing its output // to stdout and stderr. Verb, Args, and WorkingDir must be specified. diff --git a/gopls/internal/regtest/bench/iwl_test.go b/gopls/internal/regtest/bench/iwl_test.go index c4a2d0f23bf..b7e9ad30b7b 100644 --- a/gopls/internal/regtest/bench/iwl_test.go +++ b/gopls/internal/regtest/bench/iwl_test.go @@ -27,6 +27,7 @@ func BenchmarkInitialWorkspaceLoad(b *testing.B) { {"pkgsite", "internal/frontend/server.go"}, {"starlark", "starlark/eval.go"}, {"tools", "internal/lsp/cache/snapshot.go"}, + {"hashiform", "internal/provider/provider.go"}, } for _, test := range tests { diff --git a/gopls/internal/regtest/bench/repo_test.go b/gopls/internal/regtest/bench/repo_test.go index 0b92b124642..f4ea78d5c02 100644 --- a/gopls/internal/regtest/bench/repo_test.go +++ b/gopls/internal/regtest/bench/repo_test.go @@ -87,6 +87,15 @@ var repos = map[string]*repo{ short: true, inDir: flag.String("tools_dir", "", "if set, reuse this directory as x/tools@v0.9.0"), }, + + // A repo of similar size to kubernetes, but with substantially more + // complex types that led to a serious performance regression (issue #60621). + "hashiform": { + name: "hashiform", + url: "https://github.com/hashicorp/terraform-provider-aws", + commit: "ac55de2b1950972d93feaa250d7505d9ed829c7c", + inDir: flag.String("hashiform_dir", "", "if set, reuse this directory as hashiform@ac55de2"), + }, } // getRepo gets the requested repo, and skips the test if -short is set and diff --git a/internal/event/tag/tag.go b/internal/event/tag/tag.go index ff2f2ecd38d..581b26c2041 100644 --- a/internal/event/tag/tag.go +++ b/internal/event/tag/tag.go @@ -19,7 +19,7 @@ var ( File = keys.NewString("file", "") Directory = keys.New("directory", "") URI = keys.New("URI", "") - Package = keys.NewString("package", "") // Package ID + Package = keys.NewString("package", "") // sorted comma-separated list of Package IDs PackagePath = keys.NewString("package_path", "") Query = keys.New("query", "") Snapshot = keys.NewUInt64("snapshot", "") diff --git a/internal/facts/facts.go b/internal/facts/facts.go index 81df45161a8..954b42d6966 100644 --- a/internal/facts/facts.go +++ b/internal/facts/facts.go @@ -158,24 +158,52 @@ type gobFact struct { // for the same package. Each call to Decode returns an independent // fact set. type Decoder struct { - pkg *types.Package - packages map[string]*types.Package + pkg *types.Package + getPackage GetPackageFunc } // NewDecoder returns a fact decoder for the specified package. +// +// It uses a brute-force recursive approach to enumerate all objects +// defined by dependencies of pkg, so that it can learn the set of +// package paths that may be mentioned in the fact encoding. This does +// not scale well; use [NewDecoderFunc] where possible. func NewDecoder(pkg *types.Package) *Decoder { // Compute the import map for this package. // See the package doc comment. - return &Decoder{pkg, importMap(pkg.Imports())} + m := importMap(pkg.Imports()) + getPackageFunc := func(path string) *types.Package { return m[path] } + return NewDecoderFunc(pkg, getPackageFunc) +} + +// NewDecoderFunc returns a fact decoder for the specified package. +// +// It calls the getPackage function for the package path string of +// each dependency (perhaps indirect) that it encounters in the +// encoding. If the function returns nil, the fact is discarded. +// +// This function is preferred over [NewDecoder] when the client is +// capable of efficient look-up of packages by package path. +func NewDecoderFunc(pkg *types.Package, getPackage GetPackageFunc) *Decoder { + return &Decoder{ + pkg: pkg, + getPackage: getPackage, + } } -// Decode decodes all the facts relevant to the analysis of package pkg. -// The read function reads serialized fact data from an external source -// for one of of pkg's direct imports. The empty file is a valid -// encoding of an empty fact set. +// A GetPackageFunc function returns the package denoted by a package path. +type GetPackageFunc = func(pkgPath string) *types.Package + +// Decode decodes all the facts relevant to the analysis of package +// pkg. The read function reads serialized fact data from an external +// source for one of pkg's direct imports, identified by package path. +// The empty file is a valid encoding of an empty fact set. // // It is the caller's responsibility to call gob.Register on all // necessary fact types. +// +// Concurrent calls to Decode are safe, so long as the +// [GetPackageFunc] (if any) is also concurrency-safe. func (d *Decoder) Decode(read func(*types.Package) ([]byte, error)) (*Set, error) { // Read facts from imported packages. // Facts may describe indirectly imported packages, or their objects. @@ -202,13 +230,11 @@ func (d *Decoder) Decode(read func(*types.Package) ([]byte, error)) (*Set, error if err := gob.NewDecoder(bytes.NewReader(data)).Decode(&gobFacts); err != nil { return nil, fmt.Errorf("decoding facts for %q: %v", imp.Path(), err) } - if debug { - logf("decoded %d facts: %v", len(gobFacts), gobFacts) - } + logf("decoded %d facts: %v", len(gobFacts), gobFacts) // Parse each one into a key and a Fact. for _, f := range gobFacts { - factPkg := d.packages[f.PkgPath] + factPkg := d.getPackage(f.PkgPath) // possibly an indirect dependency if factPkg == nil { // Fact relates to a dependency that was // unused in this translation unit. Skip. diff --git a/internal/facts/imports.go b/internal/facts/imports.go index 7b21668660c..b18e62d1d79 100644 --- a/internal/facts/imports.go +++ b/internal/facts/imports.go @@ -21,6 +21,10 @@ import ( // Packages in the map that are only indirectly imported may be // incomplete (!pkg.Complete()). // +// This function scales very poorly with packages' transitive object +// references, which can be more than a million for each package near +// the top of a large project. (This was a significant contributor to +// #60621.) // TODO(adonovan): opt: compute this information more efficiently // by obtaining it from the internals of the gcexportdata decoder. func importMap(imports []*types.Package) map[string]*types.Package { diff --git a/internal/gcimporter/gcimporter_test.go b/internal/gcimporter/gcimporter_test.go index 6346885a58f..3db67606850 100644 --- a/internal/gcimporter/gcimporter_test.go +++ b/internal/gcimporter/gcimporter_test.go @@ -841,8 +841,7 @@ func TestExportInvalid(t *testing.T) { // Re-import it. imports := make(map[string]*types.Package) - insert := func(pkg1 *types.Package, name string) { panic("unexpected insert") } - pkg2, err := IImportShallow(fset, GetPackageFromMap(imports), data, "p", insert) + pkg2, err := IImportShallow(fset, GetPackagesFromMap(imports), data, "p") if err != nil { t.Fatalf("import: %v", err) // any failure of IExport+IImport is a bug. } diff --git a/internal/gcimporter/iexport.go b/internal/gcimporter/iexport.go index db9842706c9..eed1702186b 100644 --- a/internal/gcimporter/iexport.go +++ b/internal/gcimporter/iexport.go @@ -47,20 +47,20 @@ func IExportShallow(fset *token.FileSet, pkg *types.Package) ([]byte, error) { // IImportShallow decodes "shallow" types.Package data encoded by // IExportShallow in the same executable. This function cannot import data from // cmd/compile or gcexportdata.Write. -func IImportShallow(fset *token.FileSet, getPackage GetPackageFunc, data []byte, path string, insert InsertType) (*types.Package, error) { +// +// The importer calls getPackages to obtain package symbols for all +// packages mentioned in the export data, including the one being +// decoded. +func IImportShallow(fset *token.FileSet, getPackages GetPackagesFunc, data []byte, path string) (*types.Package, error) { const bundle = false - pkgs, err := iimportCommon(fset, getPackage, data, bundle, path, insert) + const shallow = true + pkgs, err := iimportCommon(fset, getPackages, data, bundle, path, shallow) if err != nil { return nil, err } return pkgs[0], nil } -// InsertType is the type of a function that creates a types.TypeName -// object for a named type and inserts it into the scope of the -// specified Package. -type InsertType = func(pkg *types.Package, name string) - // Current bundled export format version. Increase with each format change. // 0: initial implementation const bundleVersion = 0 @@ -1222,6 +1222,13 @@ type internalError string func (e internalError) Error() string { return "gcimporter: " + string(e) } +// TODO(adonovan): make this call panic, so that it's symmetric with errorf. +// Otherwise it's easy to forget to do anything with the error. +// +// TODO(adonovan): also, consider switching the names "errorf" and +// "internalErrorf" as the former is used for bugs, whose cause is +// internal inconsistency, whereas the latter is used for ordinary +// situations like bad input, whose cause is external. func internalErrorf(format string, args ...interface{}) error { return internalError(fmt.Sprintf(format, args...)) } diff --git a/internal/gcimporter/iimport.go b/internal/gcimporter/iimport.go index 94a5eba333f..fb6554f9261 100644 --- a/internal/gcimporter/iimport.go +++ b/internal/gcimporter/iimport.go @@ -85,7 +85,7 @@ const ( // If the export data version is not recognized or the format is otherwise // compromised, an error is returned. func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (int, *types.Package, error) { - pkgs, err := iimportCommon(fset, GetPackageFromMap(imports), data, false, path, nil) + pkgs, err := iimportCommon(fset, GetPackagesFromMap(imports), data, false, path, false) if err != nil { return 0, nil, err } @@ -94,33 +94,49 @@ func IImportData(fset *token.FileSet, imports map[string]*types.Package, data [] // IImportBundle imports a set of packages from the serialized package bundle. func IImportBundle(fset *token.FileSet, imports map[string]*types.Package, data []byte) ([]*types.Package, error) { - return iimportCommon(fset, GetPackageFromMap(imports), data, true, "", nil) + return iimportCommon(fset, GetPackagesFromMap(imports), data, true, "", false) } -// A GetPackageFunc is a function that gets the package with the given path -// from the importer state, creating it (with the specified name) if necessary. -// It is an abstraction of the map historically used to memoize package creation. +// A GetPackagesFunc function obtains the non-nil symbols for a set of +// packages, creating and recursively importing them as needed. An +// implementation should store each package symbol is in the Pkg +// field of the items array. // -// Two calls with the same path must return the same package. -// -// If the given getPackage func returns nil, the import will fail. -type GetPackageFunc = func(path, name string) *types.Package +// Any error causes importing to fail. This can be used to quickly read +// the import manifest of an export data file without fully decoding it. +type GetPackagesFunc = func(items []GetPackagesItem) error + +// A GetPackagesItem is a request from the importer for the package +// symbol of the specified name and path. +type GetPackagesItem struct { + Name, Path string + Pkg *types.Package // to be filled in by GetPackagesFunc call -// GetPackageFromMap returns a GetPackageFunc that retrieves packages from the -// given map of package path -> package. + // private importer state + pathOffset uint64 + nameIndex map[string]uint64 +} + +// GetPackagesFromMap returns a GetPackagesFunc that retrieves +// packages from the given map of package path to package. // -// The resulting func may mutate m: if a requested package is not found, a new -// package will be inserted into m. -func GetPackageFromMap(m map[string]*types.Package) GetPackageFunc { - return func(path, name string) *types.Package { - if _, ok := m[path]; !ok { - m[path] = types.NewPackage(path, name) +// The returned function may mutate m: each requested package that is not +// found is created with types.NewPackage and inserted into m. +func GetPackagesFromMap(m map[string]*types.Package) GetPackagesFunc { + return func(items []GetPackagesItem) error { + for i, item := range items { + pkg, ok := m[item.Path] + if !ok { + pkg = types.NewPackage(item.Path, item.Name) + m[item.Path] = pkg + } + items[i].Pkg = pkg } - return m[path] + return nil } } -func iimportCommon(fset *token.FileSet, getPackage GetPackageFunc, data []byte, bundle bool, path string, insert InsertType) (pkgs []*types.Package, err error) { +func iimportCommon(fset *token.FileSet, getPackages GetPackagesFunc, data []byte, bundle bool, path string, shallow bool) (pkgs []*types.Package, err error) { const currentVersion = iexportVersionCurrent version := int64(-1) if !debug { @@ -159,7 +175,7 @@ func iimportCommon(fset *token.FileSet, getPackage GetPackageFunc, data []byte, sLen := int64(r.uint64()) var fLen int64 var fileOffset []uint64 - if insert != nil { + if shallow { // Shallow mode uses a different position encoding. fLen = int64(r.uint64()) fileOffset = make([]uint64, r.uint64()) @@ -176,9 +192,9 @@ func iimportCommon(fset *token.FileSet, getPackage GetPackageFunc, data []byte, r.Seek(sLen+fLen+dLen, io.SeekCurrent) p := iimporter{ - version: int(version), - ipath: path, - insert: insert, + version: int(version), + ipath: path, + usePosv2: shallow, // precise offsets are encoded only in shallow mode stringData: stringData, stringCache: make(map[uint64]string), @@ -205,8 +221,9 @@ func iimportCommon(fset *token.FileSet, getPackage GetPackageFunc, data []byte, p.typCache[uint64(i)] = pt } - pkgList := make([]*types.Package, r.uint64()) - for i := range pkgList { + // Gather the relevant packages from the manifest. + items := make([]GetPackagesItem, r.uint64()) + for i := range items { pkgPathOff := r.uint64() pkgPath := p.stringAt(pkgPathOff) pkgName := p.stringAt(r.uint64()) @@ -215,29 +232,42 @@ func iimportCommon(fset *token.FileSet, getPackage GetPackageFunc, data []byte, if pkgPath == "" { pkgPath = path } - pkg := getPackage(pkgPath, pkgName) - if pkg == nil { - errorf("internal error: getPackage returned nil package for %s", pkgPath) - } else if pkg.Name() != pkgName { - errorf("conflicting names %s and %s for package %q", pkg.Name(), pkgName, path) - } - if i == 0 && !bundle { - p.localpkg = pkg - } - - p.pkgCache[pkgPathOff] = pkg + items[i].Name = pkgName + items[i].Path = pkgPath + items[i].pathOffset = pkgPathOff // Read index for package. nameIndex := make(map[string]uint64) nSyms := r.uint64() - // In shallow mode we don't expect an index for other packages. - assert(nSyms == 0 || p.localpkg == pkg || p.insert == nil) + // In shallow mode, only the current package (i=0) has an index. + assert(!(shallow && i > 0 && nSyms != 0)) for ; nSyms > 0; nSyms-- { name := p.stringAt(r.uint64()) nameIndex[name] = r.uint64() } - p.pkgIndex[pkg] = nameIndex + items[i].nameIndex = nameIndex + } + + // Request packages all at once from the client, + // enabling a parallel implementation. + if err := getPackages(items); err != nil { + return nil, err // don't wrap this error + } + + // Check the results and complete the index. + pkgList := make([]*types.Package, len(items)) + for i, item := range items { + pkg := item.Pkg + if pkg == nil { + errorf("internal error: getPackages returned nil package for %q", item.Path) + } else if pkg.Path() != item.Path { + errorf("internal error: getPackages returned wrong path %q, want %q", pkg.Path(), item.Path) + } else if pkg.Name() != item.Name { + errorf("internal error: getPackages returned wrong name %s for package %q, want %s", pkg.Name(), item.Path, item.Name) + } + p.pkgCache[item.pathOffset] = pkg + p.pkgIndex[pkg] = item.nameIndex pkgList[i] = pkg } @@ -308,8 +338,7 @@ type iimporter struct { version int ipath string - localpkg *types.Package - insert func(pkg *types.Package, name string) // "shallow" mode only + usePosv2 bool stringData []byte stringCache map[uint64]string @@ -357,13 +386,9 @@ func (p *iimporter) doDecl(pkg *types.Package, name string) { off, ok := p.pkgIndex[pkg][name] if !ok { - // In "shallow" mode, call back to the application to - // find the object and insert it into the package scope. - if p.insert != nil { - assert(pkg != p.localpkg) - p.insert(pkg, name) // "can't fail" - return - } + // In deep mode, the index should be complete. In shallow + // mode, we should have already recursively loaded necessary + // dependencies so the above Lookup succeeds. errorf("%v.%v not in index", pkg, name) } @@ -730,7 +755,7 @@ func (r *importReader) qualifiedIdent() (*types.Package, string) { } func (r *importReader) pos() token.Pos { - if r.p.insert != nil { // shallow mode + if r.p.usePosv2 { return r.posv2() } if r.p.version >= iexportVersionPosCol { diff --git a/internal/gcimporter/shallow_test.go b/internal/gcimporter/shallow_test.go index f73d1b39182..b775a3578de 100644 --- a/internal/gcimporter/shallow_test.go +++ b/internal/gcimporter/shallow_test.go @@ -113,30 +113,33 @@ func typecheck(t *testing.T, ppkg *packages.Package) { // importer state var ( - insert func(p *types.Package, name string) - importMap = make(map[string]*types.Package) // keys are PackagePaths - ) - loadFromExportData := func(imp *packages.Package) (*types.Package, error) { - data := []byte(imp.ExportFile) - return gcimporter.IImportShallow(fset, gcimporter.GetPackageFromMap(importMap), data, imp.PkgPath, insert) - } - insert = func(p *types.Package, name string) { - imp, ok := depsByPkgPath[p.Path()] - if !ok { - t.Fatalf("can't find dependency: %q", p.Path()) - } - imported, err := loadFromExportData(imp) - if err != nil { - t.Fatalf("unmarshal: %v", err) - } - if imported != p { - t.Fatalf("internal error: inconsistent packages") + loadFromExportData func(*packages.Package) (*types.Package, error) + importMap = map[string]*types.Package{ // keys are PackagePaths + ppkg.PkgPath: types.NewPackage(ppkg.PkgPath, ppkg.Name), } - if obj := imported.Scope().Lookup(name); obj == nil { - t.Fatalf("lookup %q.%s failed", imported.Path(), name) + ) + loadFromExportData = func(imp *packages.Package) (*types.Package, error) { + export := []byte(imp.ExportFile) + getPackages := func(items []gcimporter.GetPackagesItem) error { + for i, item := range items { + pkg, ok := importMap[item.Path] + if !ok { + dep, ok := depsByPkgPath[item.Path] + if !ok { + return fmt.Errorf("can't find dependency: %q", item.Path) + } + pkg = types.NewPackage(item.Path, dep.Name) + importMap[item.Path] = pkg + loadFromExportData(dep) // side effect: populate package scope + } + items[i].Pkg = pkg + } + return nil } + return gcimporter.IImportShallow(fset, getPackages, export, imp.PkgPath) } + // Type-check the syntax trees. cfg := &types.Config{ Error: func(e error) { t.Error(e) @@ -153,8 +156,10 @@ func typecheck(t *testing.T, ppkg *packages.Package) { }), } - // Type-check the syntax trees. - tpkg, _ := cfg.Check(ppkg.PkgPath, fset, syntax, nil) + // (Use NewChecker+Files to ensure Package.Name is set explicitly.) + tpkg := types.NewPackage(ppkg.PkgPath, ppkg.Name) + _ = types.NewChecker(cfg, fset, tpkg, nil).Files(syntax) // ignore error + // Check sanity. postTypeCheck(t, fset, tpkg) // Save the export data. From 155320d396edafbf0c588d4aa98099f9eb19b7dd Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Thu, 15 Jun 2023 16:21:26 -0400 Subject: [PATCH 04/26] gopls/internal/lsp/work: don't crash when hovering over broken mod dir Guard against modfiles without a module directive. Fixes golang/go#60821 Change-Id: I91661b2abea916db55a81323fc26dcd4f2f0516c Reviewed-on: https://go-review.googlesource.com/c/tools/+/503436 gopls-CI: kokoro TryBot-Result: Gopher Robot Run-TryBot: Robert Findley Reviewed-by: Alan Donovan --- gopls/internal/lsp/cache/mod.go | 2 +- gopls/internal/lsp/mod/diagnostics.go | 8 +++++++- gopls/internal/lsp/work/hover.go | 3 +++ gopls/internal/regtest/misc/hover_test.go | 20 ++++++++++++++++++++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/gopls/internal/lsp/cache/mod.go b/gopls/internal/lsp/cache/mod.go index 50c8b00777e..8b6331d056d 100644 --- a/gopls/internal/lsp/cache/mod.go +++ b/gopls/internal/lsp/cache/mod.go @@ -400,7 +400,7 @@ func (s *snapshot) matchErrorToModule(ctx context.Context, pm *source.ParsedModu // Show the error on the module declaration, if one exists, or // just the first line of the file. var start, end int - if pm.File.Module != nil { + if pm.File.Module != nil && pm.File.Module.Syntax != nil { syntax := pm.File.Module.Syntax start, end = syntax.Start.Byte, syntax.End.Byte } diff --git a/gopls/internal/lsp/mod/diagnostics.go b/gopls/internal/lsp/mod/diagnostics.go index cd1c85b2613..91d6c7ed90c 100644 --- a/gopls/internal/lsp/mod/diagnostics.go +++ b/gopls/internal/lsp/mod/diagnostics.go @@ -331,7 +331,13 @@ func ModVulnerabilityDiagnostics(ctx context.Context, snapshot source.Snapshot, // TODO(hyangah): place this diagnostic on the `go` directive or `toolchain` directive // after https://go.dev/issue/57001. const diagnoseStdLib = false - if diagnoseStdLib { + + // If diagnosing the stdlib, add standard library vulnerability diagnostics + // on the module declaration. + // + // Only proceed if we have a valid module declaration on which to position + // the diagnostics. + if diagnoseStdLib && pm.File.Module != nil && pm.File.Module.Syntax != nil { // Add standard library vulnerabilities. stdlibVulns := vulnsByModule["stdlib"] if len(stdlibVulns) == 0 { diff --git a/gopls/internal/lsp/work/hover.go b/gopls/internal/lsp/work/hover.go index f576d696c18..558eebc824b 100644 --- a/gopls/internal/lsp/work/hover.go +++ b/gopls/internal/lsp/work/hover.go @@ -52,6 +52,9 @@ func Hover(ctx context.Context, snapshot source.Snapshot, fh source.FileHandle, if err != nil { return nil, fmt.Errorf("getting modfile handle: %w", err) } + if pm.File.Module == nil { + return nil, fmt.Errorf("modfile has no module declaration") + } mod := pm.File.Module.Mod // Get the range to highlight for the hover. diff --git a/gopls/internal/regtest/misc/hover_test.go b/gopls/internal/regtest/misc/hover_test.go index 41c6529a7bf..9b2d86ebb69 100644 --- a/gopls/internal/regtest/misc/hover_test.go +++ b/gopls/internal/regtest/misc/hover_test.go @@ -415,3 +415,23 @@ func TestHoverLinknameDirective(t *testing.T) { } }) } + +func TestHoverGoWork_Issue60821(t *testing.T) { + const files = ` +-- go.work -- +go 1.19 + +use ( + moda + modb +) +-- moda/go.mod -- + +` + Run(t, files, func(t *testing.T, env *Env) { + env.OpenFile("go.work") + // Neither of the requests below should crash gopls. + _, _, _ = env.Editor.Hover(env.Ctx, env.RegexpSearch("go.work", "moda")) + _, _, _ = env.Editor.Hover(env.Ctx, env.RegexpSearch("go.work", "modb")) + }) +} From 6480332be24cd920bc0a15b9eb3a0cc020c1c33d Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Tue, 20 Jun 2023 15:12:10 -0400 Subject: [PATCH 05/26] gopls/internal/lsp/cache: use types.Unsafe in analysisPackage Before, the early return in typeCheck forgot to set an.types. Now the logic is pulled into single spot in the caller. This change consolidates the various places that set analysisNode.types to just two (syntax and export). Change-Id: I2f1ea6b70888bd5874f76c57443fe46b1c04ae67 Reviewed-on: https://go-review.googlesource.com/c/tools/+/504558 TryBot-Result: Gopher Robot Run-TryBot: Alan Donovan Auto-Submit: Alan Donovan gopls-CI: kokoro Reviewed-by: Robert Findley --- gopls/internal/lsp/cache/analysis.go | 30 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/gopls/internal/lsp/cache/analysis.go b/gopls/internal/lsp/cache/analysis.go index 5ba84278a0f..d9a457fd517 100644 --- a/gopls/internal/lsp/cache/analysis.go +++ b/gopls/internal/lsp/cache/analysis.go @@ -229,11 +229,6 @@ func (snapshot *snapshot) Analyze(ctx context.Context, pkgs map[PackageID]unit, allDeps: make(map[PackagePath]*analysisNode), exportDeps: make(map[PackagePath]*analysisNode), } - // Unsafe must use a distinguished types.Package. - // Mark it as fully populated from birth. - if m.PkgPath == "unsafe" { - an.typesOnce.Do(func() { an.types = types.Unsafe }) - } nodes[id] = an // -- recursion -- @@ -405,6 +400,11 @@ func (an *analysisNode) String() string { return string(an.m.ID) } // Postcondition: an.types and an.exportDeps are populated. func (an *analysisNode) _import() *types.Package { an.typesOnce.Do(func() { + if an.m.PkgPath == "unsafe" { + an.types = types.Unsafe + return + } + an.types = types.NewPackage(string(an.m.PkgPath), string(an.m.Name)) // getPackages recursively imports each dependency @@ -662,6 +662,12 @@ func (an *analysisNode) run(ctx context.Context) (*analyzeSummary, error) { // Type-check the package syntax. pkg := an.typeCheck(parsed) + // Publish the completed package. + an.typesOnce.Do(func() { an.types = pkg.types }) + if an.types != pkg.types { + log.Fatalf("typesOnce prematurely done") + } + // Compute the union of exportDeps across our direct imports. // This is the set that will be needed by the fact decoder. allExportDeps := make(map[PackagePath]*analysisNode) @@ -747,10 +753,9 @@ func (an *analysisNode) run(ctx context.Context) (*analyzeSummary, error) { }, nil } -// Postcondition: an.types and an.exportDeps are populated. +// Postcondition: analysisPackage.types and an.exportDeps are populated. func (an *analysisNode) typeCheck(parsed []*source.ParsedGoFile) *analysisPackage { m := an.m - fset := an.fset if false { // debugging log.Println("typeCheck", m.ID) @@ -758,7 +763,7 @@ func (an *analysisNode) typeCheck(parsed []*source.ParsedGoFile) *analysisPackag pkg := &analysisPackage{ m: m, - fset: fset, + fset: an.fset, parsed: parsed, files: make([]*ast.File, len(parsed)), compiles: len(m.Errors) == 0, // false => list error @@ -777,6 +782,7 @@ func (an *analysisNode) typeCheck(parsed []*source.ParsedGoFile) *analysisPackag // Unsafe has no syntax. if m.PkgPath == "unsafe" { + pkg.types = types.Unsafe return pkg } @@ -862,7 +868,7 @@ func (an *analysisNode) typeCheck(parsed []*source.ParsedGoFile) *analysisPackag // TODO(adonovan): do we actually need this?? typesinternal.SetUsesCgo(cfg) - check := types.NewChecker(cfg, fset, pkg.types, pkg.typesInfo) + check := types.NewChecker(cfg, pkg.fset, pkg.types, pkg.typesInfo) // Type checking errors are handled via the config, so ignore them here. _ = check.Files(pkg.files) @@ -874,12 +880,6 @@ func (an *analysisNode) typeCheck(parsed []*source.ParsedGoFile) *analysisPackag } } - // Publish the completed package. - an.typesOnce.Do(func() { an.types = pkg.types }) - if an.types != pkg.types { - log.Fatalf("typesOnce prematurely done") - } - // Emit the export data and compute the recursive hash. export, err := gcimporter.IExportShallow(pkg.fset, pkg.types) if err != nil { From 9960b693c44a8dfedafaca0da40a50d29e03fe42 Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Tue, 20 Jun 2023 11:33:29 -0400 Subject: [PATCH 06/26] gopls/internal/lsp/cache: guard against "unsafe" by package path, not ID The crash in golang/go#60890 suggests that a user encountered a variant of the unsafe package. I'm not sure how to reproduce this, but in any case we should be checking package path, not ID, when guarding against exporting "unsafe". For golang/go#60890 Change-Id: Ib6c546b8f74ba513f5ee3df09b5ba29cea0c1b85 Reviewed-on: https://go-review.googlesource.com/c/tools/+/504555 Run-TryBot: Robert Findley TryBot-Result: Gopher Robot gopls-CI: kokoro Reviewed-by: Alan Donovan --- gopls/internal/lsp/cache/check.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/gopls/internal/lsp/cache/check.go b/gopls/internal/lsp/cache/check.go index 81c81a30b5e..1b0844ad12d 100644 --- a/gopls/internal/lsp/cache/check.go +++ b/gopls/internal/lsp/cache/check.go @@ -453,6 +453,13 @@ func (b *typeCheckBatch) getImportPackage(ctx context.Context, id PackageID) (pk } ph := b.handles[id] + + // Do a second check for "unsafe" defensively, due to golang/go#60890. + if ph.m.PkgPath == "unsafe" { + bug.Reportf("encountered \"unsafe\" as %s (golang/go#60890)", id) + return types.Unsafe, nil + } + data, err := filecache.Get(exportDataKind, ph.key) if err == filecache.ErrNotFound { // No cached export data: type-check as fast as possible. @@ -632,13 +639,17 @@ func (b *typeCheckBatch) checkPackage(ctx context.Context, ph *packageHandle) (* diagnosticsKind: encodeDiagnostics(pkg.diagnostics), } - if ph.m.ID != "unsafe" { // unsafe cannot be exported + if ph.m.PkgPath != "unsafe" { // unsafe cannot be exported exportData, err := gcimporter.IExportShallow(pkg.fset, pkg.types) if err != nil { bug.Reportf("exporting package %v: %v", ph.m.ID, err) } else { toCache[exportDataKind] = exportData } + } else if ph.m.ID != "unsafe" { + // golang/go#60890: we should only ever see one variant of the "unsafe" + // package. + bug.Reportf("encountered \"unsafe\" as %s (golang/go#60890)", ph.m.ID) } for kind, data := range toCache { From 85554d61b94ee8b082a43399ebf894cf08d59b5a Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Tue, 20 Jun 2023 16:52:59 -0400 Subject: [PATCH 07/26] gopls/internal/lsp/cache: add debugging assertion for golang/go#60904 Updates golang/go#60904 Change-Id: Ia9f259716732705060a0d3675cd701c6c68a215f Reviewed-on: https://go-review.googlesource.com/c/tools/+/504695 gopls-CI: kokoro TryBot-Result: Gopher Robot Run-TryBot: Alan Donovan Reviewed-by: Robert Findley Auto-Submit: Alan Donovan --- gopls/internal/lsp/cache/check.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/gopls/internal/lsp/cache/check.go b/gopls/internal/lsp/cache/check.go index 1b0844ad12d..14877ff44e3 100644 --- a/gopls/internal/lsp/cache/check.go +++ b/gopls/internal/lsp/cache/check.go @@ -541,14 +541,25 @@ func (b *typeCheckBatch) importPackage(ctx context.Context, m *source.Metadata, thisPackage := types.NewPackage(string(m.PkgPath), string(m.Name)) getPackages := func(items []gcimporter.GetPackagesItem) error { for i, item := range items { + var id PackageID + var pkg *types.Package if item.Path == string(m.PkgPath) { - items[i].Pkg = thisPackage + id = m.ID + pkg = thisPackage } else { - pkg, err := b.getImportPackage(ctx, impMap[item.Path]) + id = impMap[item.Path] + var err error + pkg, err = b.getImportPackage(ctx, id) if err != nil { return err } - items[i].Pkg = pkg + } + items[i].Pkg = pkg + + // debugging issue #60904 + if pkg.Name() != item.Name { + return bug.Errorf("internal error: package name is %q, want %q (id=%q, path=%q) (see issue #60904)", + pkg.Name(), item.Name, id, item.Path) } } return nil From 0ca9d30bb602329aef3311e5e6264d6be7388410 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Wed, 21 Jun 2023 10:41:09 -0400 Subject: [PATCH 08/26] gopls/internal/lsp/cache: fix nil panic in analysis toSourceDiagnostic The set of enabled analyzers is expanded to include requirements, but we should not use the expanded set in the final loop when we map back to source.Analyzers. Nonetheless, it is surprising that any analyzer only in the expandes set should report a diagnostics, so I've added an assertion. Fixes golang/go#60909 Change-Id: I3b6e21194f7d717628822b933e6f4a387f719a1c Reviewed-on: https://go-review.googlesource.com/c/tools/+/504818 Run-TryBot: Alan Donovan gopls-CI: kokoro Reviewed-by: Robert Findley Auto-Submit: Alan Donovan TryBot-Result: Gopher Robot --- gopls/internal/lsp/cache/analysis.go | 33 ++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/gopls/internal/lsp/cache/analysis.go b/gopls/internal/lsp/cache/analysis.go index d9a457fd517..ad55f2f9776 100644 --- a/gopls/internal/lsp/cache/analysis.go +++ b/gopls/internal/lsp/cache/analysis.go @@ -176,7 +176,7 @@ func (snapshot *snapshot) Analyze(ctx context.Context, pkgs map[PackageID]unit, // Filter and sort enabled root analyzers. // A disabled analyzer may still be run if required by another. toSrc := make(map[*analysis.Analyzer]*source.Analyzer) - var enabled []*analysis.Analyzer + var enabled []*analysis.Analyzer // enabled subset + transitive requirements for _, a := range analyzers { if a.IsEnabled(snapshot.view.Options()) { toSrc[a.Analyzer] = a @@ -190,16 +190,16 @@ func (snapshot *snapshot) Analyze(ctx context.Context, pkgs map[PackageID]unit, // Register fact types of required analyzers. enabled = requiredAnalyzers(enabled) - var useFacts []*analysis.Analyzer + var facty []*analysis.Analyzer // facty subset of enabled + transitive requirements for _, a := range enabled { if len(a.FactTypes) > 0 { - useFacts = append(useFacts, a) + facty = append(facty, a) for _, f := range a.FactTypes { gob.Register(f) // <2us } } } - useFacts = requiredAnalyzers(useFacts) + facty = requiredAnalyzers(facty) // File set for this batch (entire graph) of analysis. fset := token.NewFileSet() @@ -208,7 +208,9 @@ func (snapshot *snapshot) Analyze(ctx context.Context, pkgs map[PackageID]unit, // build the DAG of packages we're going to analyze. // // Root nodes will run the enabled set of analyzers, - // whereas dependencies will run only the useFacts set. + // whereas dependencies will run only the facty set. + // Because (by construction) enabled is a superset of facty, + // we can analyze each node with exactly one set of analyzers. nodes := make(map[PackageID]*analysisNode) var leaves []*analysisNode // nodes with no unfinished successors var makeNode func(from *analysisNode, id PackageID) (*analysisNode, error) @@ -225,7 +227,7 @@ func (snapshot *snapshot) Analyze(ctx context.Context, pkgs map[PackageID]unit, an = &analysisNode{ fset: fset, m: m, - analyzers: useFacts, // all nodes run at least the facty analyzers + analyzers: facty, // all nodes run at least the facty analyzers allDeps: make(map[PackagePath]*analysisNode), exportDeps: make(map[PackagePath]*analysisNode), } @@ -338,6 +340,23 @@ func (snapshot *snapshot) Analyze(ctx context.Context, pkgs map[PackageID]unit, var results []*source.Diagnostic for _, root := range roots { for _, a := range enabled { + // Skip analyzers that were added only to + // fulfil requirements of the original set. + srcAnalyzer, ok := toSrc[a] + if !ok { + // Although this 'skip' operation is logically sound, + // it is nonetheless surprising that its absence should + // cause #60909 since none of the analyzers added for + // requirements (e.g. ctrlflow, inspect, buildssa) + // is capable of reporting diagnostics. + if summary := root.summary.Actions[a.Name]; summary != nil { + if n := len(summary.Diagnostics); n > 0 { + bug.Reportf("Internal error: got %d unexpected diagnostics from analyzer %s. This analyzer was added only to fulfil the requirements of the requested set of analyzers, and it is not expected that such analyzers report diagnostics. Please report this in issue #60909.", n, a) + } + } + continue + } + // Inv: root.summary is the successful result of run (via runCached). summary, ok := root.summary.Actions[a.Name] if summary == nil { @@ -348,7 +367,7 @@ func (snapshot *snapshot) Analyze(ctx context.Context, pkgs map[PackageID]unit, continue // action failed } for _, gobDiag := range summary.Diagnostics { - results = append(results, toSourceDiagnostic(toSrc[a], &gobDiag)) + results = append(results, toSourceDiagnostic(srcAnalyzer, &gobDiag)) } } } From 63624dfc81c499476bef6cb30baae98fcf5b8f0b Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Wed, 14 Jun 2023 11:08:30 -0400 Subject: [PATCH 09/26] gopls/internal/lsp/cache: remove ITVs from shared import graph The shared import graph exists to optimize type-checking of open packages following a change. But we should only ever ask for ITVs when type-checking the entire workspace (in the debounced call to diagnoseSnapshot). For golang/go#53992 Change-Id: Ie968d4b2a0476822999aa216c678bed370aaa67b Reviewed-on: https://go-review.googlesource.com/c/tools/+/503316 Run-TryBot: Robert Findley TryBot-Result: Gopher Robot Reviewed-by: Alan Donovan gopls-CI: kokoro --- gopls/internal/lsp/cache/check.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gopls/internal/lsp/cache/check.go b/gopls/internal/lsp/cache/check.go index 14877ff44e3..9329d4bd0d1 100644 --- a/gopls/internal/lsp/cache/check.go +++ b/gopls/internal/lsp/cache/check.go @@ -205,7 +205,11 @@ func (s *snapshot) resolveImportGraph() (*importGraph, error) { openPackages := make(map[PackageID]bool) for _, fh := range s.overlays() { for _, id := range meta.ids[fh.URI()] { - openPackages[id] = true + // TODO(rfindley): remove this defensiveness after the release. We can + // rely on m.metadata[id] != nil. + if m := meta.metadata[id]; m != nil && !m.IsIntermediateTestVariant() { + openPackages[id] = true + } } } From 87df9bcac8d968740c8a22f1a2210cae39501a4c Mon Sep 17 00:00:00 2001 From: Heschi Kreinick Date: Wed, 21 Jun 2023 16:23:23 -0400 Subject: [PATCH 10/26] cmd/godoc: disable TestWebIndex, a very slow test It's timing out when run under the race detector. Change-Id: I40b0fb669f4bdb0f2110856146786176235c6653 Reviewed-on: https://go-review.googlesource.com/c/tools/+/504995 TryBot-Result: Gopher Robot Run-TryBot: Heschi Kreinick Reviewed-by: Robert Findley --- cmd/godoc/godoc_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/godoc/godoc_test.go b/cmd/godoc/godoc_test.go index 63df8974978..b2ebe8581a6 100644 --- a/cmd/godoc/godoc_test.go +++ b/cmd/godoc/godoc_test.go @@ -187,6 +187,7 @@ func TestWeb(t *testing.T) { // Basic integration test for godoc HTTP interface. func TestWebIndex(t *testing.T) { + t.Skip("slow test of to-be-deleted code (golang/go#59056)") if testing.Short() { t.Skip("skipping slow test in -short mode") } From c8278fe909aba46f3cad6253a0d3b1303a16ad72 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Thu, 22 Jun 2023 16:36:59 -0400 Subject: [PATCH 11/26] gopls/internal/lsp/cache: don't record edges to invalid packages The importer assumes that packages have non-empty package path and name. Enforce this invariant when loading metadata. Fixes golang/go#60952 Change-Id: I2f4f18e475ba306d93c8b649d19897a584e5ba19 Reviewed-on: https://go-review.googlesource.com/c/tools/+/505219 TryBot-Result: Gopher Robot Reviewed-by: Alan Donovan Run-TryBot: Robert Findley gopls-CI: kokoro --- go/packages/golist.go | 3 +++ gopls/internal/lsp/cache/load.go | 14 +++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/go/packages/golist.go b/go/packages/golist.go index e84f19dfa98..58230038a7c 100644 --- a/go/packages/golist.go +++ b/go/packages/golist.go @@ -671,6 +671,9 @@ func (state *golistState) createDriverResponse(words ...string) (*driverResponse // Temporary work-around for golang/go#39986. Parse filenames out of // error messages. This happens if there are unrecoverable syntax // errors in the source, so we can't match on a specific error message. + // + // TODO(rfindley): remove this heuristic, in favor of considering + // InvalidGoFiles from the list driver. if err := p.Error; err != nil && state.shouldAddFilenameFromError(p) { addFilenameFromPos := func(pos string) bool { split := strings.Split(pos, ":") diff --git a/gopls/internal/lsp/cache/load.go b/gopls/internal/lsp/cache/load.go index da985a8ea38..eb302b7b332 100644 --- a/gopls/internal/lsp/cache/load.go +++ b/gopls/internal/lsp/cache/load.go @@ -528,9 +528,21 @@ func buildMetadata(updates map[PackageID]*source.Metadata, pkg *packages.Package continue } + buildMetadata(updates, imported, loadDir, false) // only top level packages can be standalone + + // Don't record edges to packages with no name, as they cause trouble for + // the importer (golang/go#60952). + // + // However, we do want to insert these packages into the update map + // (buildMetadata above), so that we get type-checking diagnostics for the + // invalid packages. + if imported.Name == "" { + depsByImpPath[importPath] = "" // missing + continue + } + depsByImpPath[importPath] = PackageID(imported.ID) depsByPkgPath[PackagePath(imported.PkgPath)] = PackageID(imported.ID) - buildMetadata(updates, imported, loadDir, false) // only top level packages can be standalone } m.DepsByImpPath = depsByImpPath m.DepsByPkgPath = depsByPkgPath From fa103593d02f86f400531e4294b9b294d4893901 Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Fri, 23 Jun 2023 09:32:56 -0400 Subject: [PATCH 12/26] gopls/internal/lsp/cache: don't panic when import fails during analysis We have received a number of empty crash reports for gopls since cutting the v0.12.3 release. We don't want to auto-populate crash reports with log.Fatal messages (as unlike panicking stacks they may contain the name of user packages), so in almost all cases we lose information as to why gopls crashed (users are unlikely to pre-populate this information). In general, I don't think failing analysis should crash the process, so switch this failure mode to a bug report. Fixes golang/go#60963 Change-Id: I670ee4b1adbe9f37d763c5684b14d4bcb78dcb63 Reviewed-on: https://go-review.googlesource.com/c/tools/+/505575 gopls-CI: kokoro Reviewed-by: Alan Donovan Run-TryBot: Robert Findley TryBot-Result: Gopher Robot --- gopls/internal/lsp/cache/analysis.go | 36 ++++++++++++++++++---------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/gopls/internal/lsp/cache/analysis.go b/gopls/internal/lsp/cache/analysis.go index ad55f2f9776..52e730b65fb 100644 --- a/gopls/internal/lsp/cache/analysis.go +++ b/gopls/internal/lsp/cache/analysis.go @@ -408,8 +408,10 @@ type analysisNode struct { exportDeps map[PackagePath]*analysisNode // subset of allDeps ref'd by export data (+self) count int32 // number of unfinished successors summary *analyzeSummary // serializable result of analyzing this package - typesOnce sync.Once // guards lazy population of types field - types *types.Package // type information lazily imported from summary + + typesOnce sync.Once // guards lazy population of types and typesErr fields + types *types.Package // type information lazily imported from summary + typesErr error // an error producing type information } func (an *analysisNode) String() string { return string(an.m.ID) } @@ -417,7 +419,7 @@ func (an *analysisNode) String() string { return string(an.m.ID) } // _import imports this node's types.Package from export data, if not already done. // Precondition: analysis was a success. // Postcondition: an.types and an.exportDeps are populated. -func (an *analysisNode) _import() *types.Package { +func (an *analysisNode) _import() (*types.Package, error) { an.typesOnce.Do(func() { if an.m.PkgPath == "unsafe" { an.types = types.Unsafe @@ -439,12 +441,19 @@ func (an *analysisNode) _import() *types.Package { } an.exportDeps[path] = dep // record, for later fact decoding if dep == an { - items[i].Pkg = an.types + if an.typesErr != nil { + return an.typesErr + } else { + items[i].Pkg = an.types + } } else { i := i g.Go(func() error { - items[i].Pkg = dep._import() - return nil + depPkg, err := dep._import() + if err == nil { + items[i].Pkg = depPkg + } + return err }) } } @@ -452,13 +461,13 @@ func (an *analysisNode) _import() *types.Package { } pkg, err := gcimporter.IImportShallow(an.fset, getPackages, an.summary.Export, string(an.m.PkgPath)) if err != nil { - log.Fatalf("%s: invalid export data: %v", an.m, err) - } - if pkg != an.types { + an.typesErr = bug.Errorf("%s: invalid export data: %v", an.m, err) + an.types = nil + } else if pkg != an.types { log.Fatalf("%s: inconsistent packages", an.m) } }) - return an.types + return an.types, an.typesErr } // analyzeSummary is a gob-serializable summary of successfully @@ -703,7 +712,10 @@ func (an *analysisNode) run(ctx context.Context) (*analyzeSummary, error) { // Does the fact relate to a package referenced by export data? if dep, ok := allExportDeps[PackagePath(path)]; ok { dep.typesOnce.Do(func() { log.Fatal("dep.types not populated") }) - return dep.types + if dep.typesErr == nil { + return dep.types + } + return nil } // If the fact relates to a dependency not referenced @@ -867,7 +879,7 @@ func (an *analysisNode) typeCheck(parsed []*source.ParsedGoFile) *analysisPackag return nil, fmt.Errorf("invalid use of internal package %s", importPath) } - return dep._import(), nil + return dep._import() }), } From 8f9bce43b33fe41595fc72852efefcb52a099c18 Mon Sep 17 00:00:00 2001 From: Lasse Folger Date: Fri, 23 Jun 2023 12:37:29 +0200 Subject: [PATCH 13/26] go/ast/inspector: clarify behavior regarding comment traversal The inspector does not walk all ast.Comments nodes but only those attached to declarations (i.e. doc comments). Clarify this in the the documentation of the traversal functions. Change-Id: I5cc9c5914fc8a7f0687ae632ba884d1317a4a7cb Reviewed-on: https://go-review.googlesource.com/c/tools/+/505495 Auto-Submit: Alan Donovan TryBot-Result: Gopher Robot gopls-CI: kokoro Reviewed-by: Alan Donovan Run-TryBot: Alan Donovan --- go/ast/inspector/inspector.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/go/ast/inspector/inspector.go b/go/ast/inspector/inspector.go index 3fbfebf3693..1fc1de0bd10 100644 --- a/go/ast/inspector/inspector.go +++ b/go/ast/inspector/inspector.go @@ -64,8 +64,9 @@ type event struct { // depth-first order. It calls f(n) for each node n before it visits // n's children. // +// The complete traversal sequence is determined by ast.Inspect. // The types argument, if non-empty, enables type-based filtering of -// events. The function f if is called only for nodes whose type +// events. The function f is called only for nodes whose type // matches an element of the types slice. func (in *Inspector) Preorder(types []ast.Node, f func(ast.Node)) { // Because it avoids postorder calls to f, and the pruning @@ -97,6 +98,7 @@ func (in *Inspector) Preorder(types []ast.Node, f func(ast.Node)) { // of the non-nil children of the node, followed by a call of // f(n, false). // +// The complete traversal sequence is determined by ast.Inspect. // The types argument, if non-empty, enables type-based filtering of // events. The function f if is called only for nodes whose type // matches an element of the types slice. From 9d8d4089ca1bf8aa2bf11dbffa5803f76cf0968d Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Fri, 23 Jun 2023 11:50:36 -0400 Subject: [PATCH 14/26] gopls/internal/lsp/cmd: list bug reports in 'gopls bug' This change causes gopls bug to print cached bug reports to stdout, and to summarize them (by file:line) in the GitHub Issue form. We don't post the bug contents, but we invite the user to copy them. Also, move the undocumented bug injection (for testing) from 'gopls stats' to 'gopls bug'. Updates golang/go#60969 Change-Id: I19648e978dca3b74954d17ac449e22537c5f1e02 Reviewed-on: https://go-review.googlesource.com/c/tools/+/505579 Reviewed-by: Hyang-Ah Hana Kim Run-TryBot: Alan Donovan TryBot-Result: Gopher Robot --- gopls/internal/lsp/cmd/info.go | 74 ++++++++++++++++++- gopls/internal/lsp/cmd/stats.go | 8 -- .../internal/lsp/cmd/test/integration_test.go | 6 +- gopls/internal/lsp/filecache/filecache.go | 2 +- 4 files changed, 74 insertions(+), 16 deletions(-) diff --git a/gopls/internal/lsp/cmd/info.go b/gopls/internal/lsp/cmd/info.go index 68ef40ffb29..d63e24b6bfc 100644 --- a/gopls/internal/lsp/cmd/info.go +++ b/gopls/internal/lsp/cmd/info.go @@ -12,10 +12,13 @@ import ( "fmt" "net/url" "os" + "sort" "strings" + goplsbug "golang.org/x/tools/gopls/internal/bug" "golang.org/x/tools/gopls/internal/lsp/browser" "golang.org/x/tools/gopls/internal/lsp/debug" + "golang.org/x/tools/gopls/internal/lsp/filecache" "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/internal/tool" ) @@ -128,10 +131,50 @@ A failing unit test is the best. // Run collects some basic information and then prepares an issue ready to // be reported. func (b *bug) Run(ctx context.Context, args ...string) error { - buf := &bytes.Buffer{} - fmt.Fprint(buf, goplsBugHeader) - debug.PrintVersionInfo(ctx, buf, true, debug.Markdown) - body := buf.String() + // This undocumented environment variable allows + // the cmd integration test (and maintainers) to + // trigger a call to bug.Report. + if msg := os.Getenv("TEST_GOPLS_BUG"); msg != "" { + filecache.Start() // register bug handler + goplsbug.Report(msg) + return nil + } + + // Enumerate bug reports, grouped and sorted. + _, reports := filecache.BugReports() + sort.Slice(reports, func(i, j int) bool { + x, y := reports[i], reports[i] + if x.Key != y.Key { + return x.Key < y.Key // ascending key order + } + return y.AtTime.Before(x.AtTime) // most recent first + }) + keyDenom := make(map[string]int) // key is "file:line" + for _, report := range reports { + keyDenom[report.Key]++ + } + + // Privacy: the content of 'public' will be posted to GitHub + // to populate an issue textarea. Even though the user must + // submit the form to share the information with the world, + // merely populating the form causes us to share the + // information with GitHub itself. + // + // For that reason, we cannot write private information to + // public, such as bug reports, which may quote source code. + public := &bytes.Buffer{} + fmt.Fprint(public, goplsBugHeader) + if len(reports) > 0 { + fmt.Fprintf(public, "#### Internal errors\n\n") + fmt.Fprintf(public, "Gopls detected %d internal errors, %d distinct:\n", + len(reports), len(keyDenom)) + for key, denom := range keyDenom { + fmt.Fprintf(public, "- %s (%d)\n", key, denom) + } + fmt.Fprintf(public, "\nPlease copy the full information printed by `gopls bug` here, if you are comfortable sharing it.\n\n") + } + debug.PrintVersionInfo(ctx, public, true, debug.Markdown) + body := public.String() title := strings.Join(args, " ") if !strings.HasPrefix(title, goplsBugPrefix) { title = goplsBugPrefix + title @@ -140,6 +183,29 @@ func (b *bug) Run(ctx context.Context, args ...string) error { fmt.Print("Please file a new issue at golang.org/issue/new using this template:\n\n") fmt.Print(body) } + + // Print bug reports to stdout (not GitHub). + keyNum := make(map[string]int) + for _, report := range reports { + fmt.Printf("-- %v -- \n", report.AtTime) + + // Append seq number (e.g. " (1/2)") for repeated keys. + var seq string + if denom := keyDenom[report.Key]; denom > 1 { + keyNum[report.Key]++ + seq = fmt.Sprintf(" (%d/%d)", keyNum[report.Key], denom) + } + + // Privacy: + // - File and Stack may contain the name of the user that built gopls. + // - Description may contain names of the user's packages/files/symbols. + fmt.Printf("%s:%d: %s%s\n\n", report.File, report.Line, report.Description, seq) + fmt.Printf("%s\n\n", report.Stack) + } + if len(reports) > 0 { + fmt.Printf("Please copy the above information into the GitHub issue, if you are comfortable sharing it.\n") + } + return nil } diff --git a/gopls/internal/lsp/cmd/stats.go b/gopls/internal/lsp/cmd/stats.go index a681c5c0642..4986107134e 100644 --- a/gopls/internal/lsp/cmd/stats.go +++ b/gopls/internal/lsp/cmd/stats.go @@ -57,14 +57,6 @@ Example: } func (s *stats) Run(ctx context.Context, args ...string) error { - // This undocumented environment variable allows - // the cmd integration test to trigger a call to bug.Report. - if msg := os.Getenv("TEST_GOPLS_BUG"); msg != "" { - filecache.Start() // effect: register bug handler - goplsbug.Report(msg) - return nil - } - if s.app.Remote != "" { // stats does not work with -remote. // Other sessions on the daemon may interfere with results. diff --git a/gopls/internal/lsp/cmd/test/integration_test.go b/gopls/internal/lsp/cmd/test/integration_test.go index 52ecb239035..91c214c6751 100644 --- a/gopls/internal/lsp/cmd/test/integration_test.go +++ b/gopls/internal/lsp/cmd/test/integration_test.go @@ -702,7 +702,7 @@ package foo oops := fmt.Sprintf("oops-%d", rand.Int()) { env := []string{"TEST_GOPLS_BUG=" + oops} - res := goplsWithEnv(t, tree, env, "stats") + res := goplsWithEnv(t, tree, env, "bug") res.checkExit(true) } @@ -745,8 +745,8 @@ package foo { got := fmt.Sprint(stats.BugReports) wants := []string{ - "cmd/stats.go", // File containing call to bug.Report - oops, // Description + "cmd/info.go", // File containing call to bug.Report + oops, // Description } for _, want := range wants { if !strings.Contains(got, want) { diff --git a/gopls/internal/lsp/filecache/filecache.go b/gopls/internal/lsp/filecache/filecache.go index 47ee89e41b0..624f8722837 100644 --- a/gopls/internal/lsp/filecache/filecache.go +++ b/gopls/internal/lsp/filecache/filecache.go @@ -552,7 +552,7 @@ func init() { // used by this process (or "" on initialization error). func BugReports() (string, []bug.Bug) { // To test this logic, run: - // $ TEST_GOPLS_BUG=oops gopls stats # trigger a bug + // $ TEST_GOPLS_BUG=oops gopls bug # trigger a bug // $ gopls stats # list the bugs dir, err := getCacheDir() From a6594121c294b66bccc12ed8298411891f03fd36 Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Sun, 27 Jan 2019 13:55:40 -0500 Subject: [PATCH 15/26] go/vcs: deprecate package in favor of go list -json This package has diverged significantly from actual cmd/go import path resolution behavior. The recommended course of action is for people to start using the go command itself, where this logic is guaranteed to be up to date. cmd/go also has support for modules, while this package does not. I've considered two alternatives to deprecating this package: 1. Not deprecate it, keep it as is. This is not a good option because the package deviates from cmd/go import path resolution behavior and doesn't have all security fixes backported to it. Keeping it in this state without deprecating it can be misleading, since people may think this package implements the stated goal of matching cmd/go behavior and is well supported, which is not the current reality. 2. Not deprecate it, try to make it up to date with cmd/go import path resolution logic. This is not a good option because VCSs are no longer guaranteed to exist for packages located inside modules. To expose the import path to module resolution logic, the API of this package would need to be significantly modified. At this time, my understanding is such work is not in scope and people are encouraged to use the existing go command instead. In 2019, when this CL was mailed, deprecating seemed as the best option. In 2023, when this CL was merged, deprecating seemed as the best option. Fixes golang/go#11490. For golang/go#57051. Change-Id: Id32d2bec5706c4e87126b825de5215fa5d1ba8ac Reviewed-on: https://go-review.googlesource.com/c/tools/+/159818 gopls-CI: kokoro TryBot-Result: Gopher Robot Auto-Submit: Dmitri Shuralyov Reviewed-by: Dmitri Shuralyov Reviewed-by: Bryan Mills Run-TryBot: Dmitri Shuralyov --- go/vcs/vcs.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/go/vcs/vcs.go b/go/vcs/vcs.go index 54d850535d2..edbdf60827a 100644 --- a/go/vcs/vcs.go +++ b/go/vcs/vcs.go @@ -6,11 +6,17 @@ // and using version control systems, which can be used to // implement behavior similar to the standard "go get" command. // -// This package is a copy of internal code in package cmd/go/internal/get, -// modified to make the identifiers exported. It's provided here -// for developers who want to write tools with similar semantics. -// It needs to be manually kept in sync with upstream when changes are -// made to cmd/go/internal/get; see https://golang.org/issue/11490. +// Deprecated: Use the go list command with -json flag instead, +// which implements up-to-date import path resolution behavior, +// module support, and includes the latest security fixes. +// +// This package was a copy of internal code in package cmd/go/internal/get +// before module support, modified to make the identifiers exported. +// It was provided here for developers who wanted to write tools with similar semantics. +// It needed to be manually kept in sync with upstream when changes were +// made to cmd/go/internal/get, as tracked in go.dev/issue/11490. +// By now, it has diverged significantly from upstream cmd/go/internal/get +// behavior and is not being actively updated. package vcs // import "golang.org/x/tools/go/vcs" import ( From d03a819dc0620bc996dc43fa7762619faf44d58e Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Sat, 24 Jun 2023 21:22:46 -0400 Subject: [PATCH 16/26] go/vcs: isolate into a nested module The previous change marked the vcs package as deprecated. This change isolates it into a nested module where it will be tagged as v0.1.0-deprecated, its last version. The next change deletes the nested module from the tools repo. For golang/go#57051. Change-Id: I0b7df60dbe87c1d97f150e5f8ca10e9d281a9364 Reviewed-on: https://go-review.googlesource.com/c/tools/+/505955 gopls-CI: kokoro Run-TryBot: Dmitri Shuralyov Auto-Submit: Dmitri Shuralyov Reviewed-by: Dmitri Shuralyov Reviewed-by: Bryan Mills TryBot-Result: Gopher Robot --- go/vcs/go.mod | 7 +++++++ go/vcs/go.sum | 2 ++ go/vcs/vcs.go | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 go/vcs/go.mod create mode 100644 go/vcs/go.sum diff --git a/go/vcs/go.mod b/go/vcs/go.mod new file mode 100644 index 00000000000..74da6cb1e3f --- /dev/null +++ b/go/vcs/go.mod @@ -0,0 +1,7 @@ +// Deprecated: This module contains one deprecated package. +// See the package deprecation notice for more information. +module golang.org/x/tools/go/vcs + +go 1.19 + +require golang.org/x/sys v0.9.0 diff --git a/go/vcs/go.sum b/go/vcs/go.sum new file mode 100644 index 00000000000..cb35c1fbf4d --- /dev/null +++ b/go/vcs/go.sum @@ -0,0 +1,2 @@ +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/go/vcs/vcs.go b/go/vcs/vcs.go index edbdf60827a..232177dcf18 100644 --- a/go/vcs/vcs.go +++ b/go/vcs/vcs.go @@ -16,7 +16,7 @@ // It needed to be manually kept in sync with upstream when changes were // made to cmd/go/internal/get, as tracked in go.dev/issue/11490. // By now, it has diverged significantly from upstream cmd/go/internal/get -// behavior and is not being actively updated. +// behavior and will not receive any further updates. package vcs // import "golang.org/x/tools/go/vcs" import ( From 5a89a3bf267ef12790327b8692c88654845bc78d Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Sat, 24 Jun 2023 20:50:29 -0400 Subject: [PATCH 17/26] go/vcs: delete This will also close issues 10159, 12079, 19362, and 22556 as 'not planned'. Fixes golang/go#57051. Change-Id: I1cc6851a60c554d2266b5c03b8c010270919f5be Reviewed-on: https://go-review.googlesource.com/c/tools/+/505956 Run-TryBot: Dmitri Shuralyov TryBot-Result: Gopher Robot gopls-CI: kokoro Auto-Submit: Dmitri Shuralyov Reviewed-by: Bryan Mills Reviewed-by: Dmitri Shuralyov --- README.md | 3 +- go/vcs/discovery.go | 83 ----- go/vcs/env.go | 39 --- go/vcs/go.mod | 7 - go/vcs/go.sum | 2 - go/vcs/http.go | 80 ----- go/vcs/vcs.go | 764 -------------------------------------------- go/vcs/vcs_test.go | 309 ------------------ 8 files changed, 1 insertion(+), 1286 deletions(-) delete mode 100644 go/vcs/discovery.go delete mode 100644 go/vcs/env.go delete mode 100644 go/vcs/go.mod delete mode 100644 go/vcs/go.sum delete mode 100644 go/vcs/http.go delete mode 100644 go/vcs/vcs.go delete mode 100644 go/vcs/vcs_test.go diff --git a/README.md b/README.md index 0a2fcf18eb4..4dbf6d69aed 100644 --- a/README.md +++ b/README.md @@ -62,12 +62,11 @@ Selected packages: Numerous other packages provide more esoteric functionality. -