From 0c6d66dcf071a9067d4e1893b580d97768ab989e Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Thu, 27 Feb 2025 15:24:45 -0700 Subject: [PATCH 01/20] starting migration --- build/build.go | 45 ++++++++++++++++++++---- compiler/compiler_test.go | 1 + compiler/{ => linkname}/linkname.go | 2 +- compiler/{ => linkname}/linkname_test.go | 2 +- compiler/package.go | 23 +----------- compiler/sources/sources.go | 1 + 6 files changed, 43 insertions(+), 31 deletions(-) rename compiler/{ => linkname}/linkname.go (99%) rename compiler/{ => linkname}/linkname_test.go (99%) diff --git a/build/build.go b/build/build.go index 4517979c0..3fb53b326 100644 --- a/build/build.go +++ b/build/build.go @@ -27,9 +27,11 @@ import ( "github.com/fsnotify/fsnotify" "github.com/gopherjs/gopherjs/compiler" "github.com/gopherjs/gopherjs/compiler/astutil" + "github.com/gopherjs/gopherjs/compiler/internal/typeparams" "github.com/gopherjs/gopherjs/compiler/jsFile" "github.com/gopherjs/gopherjs/compiler/sources" "github.com/gopherjs/gopherjs/internal/errorList" + "github.com/gopherjs/gopherjs/internal/experiments" "github.com/gopherjs/gopherjs/internal/testmain" log "github.com/sirupsen/logrus" @@ -759,7 +761,7 @@ type Session struct { // sources is a map of parsed packages that have been built and augmented. // This is keyed using resolved import paths. This is used to avoid // rebuilding and augmenting packages that are imported by several packages. - // These sources haven't been sorted nor simplified yet. + // These sources haven't been simplified yet. sources map[string]*sources.Sources // Binary archives produced during the current session and assumed to be @@ -918,7 +920,7 @@ func (s *Session) BuildProject(pkg *PackageData) (*compiler.Archive, error) { // ensure that runtime for gopherjs is imported pkg.Imports = append(pkg.Imports, `runtime`) - // Build the project to get the sources for the parsed packages. + // Load the project to get the sources for the parsed packages. var srcs *sources.Sources var err error if pkg.IsTest { @@ -930,12 +932,12 @@ func (s *Session) BuildProject(pkg *PackageData) (*compiler.Archive, error) { return nil, err } - // TODO(grantnelson-wf): At this point we have all the parsed packages we - // need to compile the whole project, including testmain, if needed. - // We can perform analysis on the whole project at this point to propagate - // flatten, blocking, etc. information and check types to get the type info - // with all the instances for all generics in the whole project. + // Prepare and analyze the source code for compiling. + if err := s.prepareSources(); err != nil { + return nil, err + } + // Compile the project into Archives containing the generated JS. return s.compilePackages(srcs) } @@ -1089,6 +1091,7 @@ func (s *Session) loadPackages(pkg *PackageData) (*sources.Sources, error) { FileSet: fileSet, JSFiles: append(pkg.JSFiles, overlayJsFiles...), } + srcs.Sort() s.sources[pkg.ImportPath] = srcs // Import dependencies from the augmented files, @@ -1103,6 +1106,34 @@ func (s *Session) loadPackages(pkg *PackageData) (*sources.Sources, error) { return srcs, nil } +func (s *Session) prepareSources() error { + for importPath, srcs := range s.sources { + + tContext := types.NewContext() + typesInfo, typesPkg, err := srcs.TypeCheck(importContext, sizes32, tContext) + if err != nil { + return err + } + if genErr := typeparams.RequiresGenericsSupport(typesInfo); genErr != nil && !experiments.Env.Generics { + return fmt.Errorf("package %s requires generics support (https://github.com/gopherjs/gopherjs/issues/1013): %w", srcs.ImportPath, genErr) + } + s.Types[importPath] = typesPkg + + // Extract all go:linkname compiler directives from the package source. + goLinknames, err := parseAllGoLinknames(srcs) + if err != nil { + return err + } + + srcs = srcs.Simplified(typesInfo) + s.sources[importPath] = srcs + + rootCtx := newRootCtx(tContext, srcs, typesInfo, typesPkg, importContext.isBlocking, minify) + + } + return nil +} + func (s *Session) compilePackages(srcs *sources.Sources) (*compiler.Archive, error) { if archive, ok := s.UpToDateArchives[srcs.ImportPath]; ok { return archive, nil diff --git a/compiler/compiler_test.go b/compiler/compiler_test.go index 4c1dd4dba..1a6453566 100644 --- a/compiler/compiler_test.go +++ b/compiler/compiler_test.go @@ -699,6 +699,7 @@ func compileProject(t *testing.T, root *packages.Package, minify bool) map[strin Files: pkg.Syntax, FileSet: pkg.Fset, } + srcs.Sort() // compile package a, err := Compile(srcs, importContext, minify) diff --git a/compiler/linkname.go b/compiler/linkname/linkname.go similarity index 99% rename from compiler/linkname.go rename to compiler/linkname/linkname.go index af3ef6cc0..32e01932c 100644 --- a/compiler/linkname.go +++ b/compiler/linkname/linkname.go @@ -1,4 +1,4 @@ -package compiler +package linkname import ( "fmt" diff --git a/compiler/linkname_test.go b/compiler/linkname/linkname_test.go similarity index 99% rename from compiler/linkname_test.go rename to compiler/linkname/linkname_test.go index 7f46c6cfb..f8c7a104f 100644 --- a/compiler/linkname_test.go +++ b/compiler/linkname/linkname_test.go @@ -1,4 +1,4 @@ -package compiler +package linkname import ( "go/ast" diff --git a/compiler/package.go b/compiler/package.go index 239c9bf75..90de230a8 100644 --- a/compiler/package.go +++ b/compiler/package.go @@ -15,7 +15,6 @@ import ( "github.com/gopherjs/gopherjs/compiler/sources" "github.com/gopherjs/gopherjs/compiler/typesutil" "github.com/gopherjs/gopherjs/internal/errorList" - "github.com/gopherjs/gopherjs/internal/experiments" ) // pkgContext maintains compiler context for a specific package. @@ -243,27 +242,7 @@ func Compile(srcs sources.Sources, importContext *ImportContext, minify bool) (_ err = bailout(fmt.Errorf("unexpected compiler panic while building package %q: %v", srcs.ImportPath, e)) }() - srcs.Sort() - - tContext := types.NewContext() - typesInfo, typesPkg, err := srcs.TypeCheck(importContext, sizes32, tContext) - if err != nil { - return nil, err - } - if genErr := typeparams.RequiresGenericsSupport(typesInfo); genErr != nil && !experiments.Env.Generics { - return nil, fmt.Errorf("package %s requires generics support (https://github.com/gopherjs/gopherjs/issues/1013): %w", srcs.ImportPath, genErr) - } - importContext.Packages[srcs.ImportPath] = typesPkg - - // Extract all go:linkname compiler directives from the package source. - goLinknames, err := parseAllGoLinknames(srcs) - if err != nil { - return nil, err - } - - srcs = srcs.Simplified(typesInfo) - - rootCtx := newRootCtx(tContext, srcs, typesInfo, typesPkg, importContext.isBlocking, minify) + // TODO(grantnelson-wf): Clean up this function and fetch information importedPaths, importDecls := rootCtx.importDecls() diff --git a/compiler/sources/sources.go b/compiler/sources/sources.go index fbcc98b95..c23ebae43 100644 --- a/compiler/sources/sources.go +++ b/compiler/sources/sources.go @@ -68,6 +68,7 @@ func (s Sources) Simplified(typesInfo *types.Info) Sources { for i, file := range s.Files { simplified.Files[i] = astrewrite.Simplify(file, typesInfo, false) } + simplified.Sort() return simplified } From 4c537e7e94eec873c4b5f90f18827a09bf7343ff Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Thu, 27 Feb 2025 15:25:46 -0700 Subject: [PATCH 02/20] starting migration --- compiler/package.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compiler/package.go b/compiler/package.go index 90de230a8..f96573a6f 100644 --- a/compiler/package.go +++ b/compiler/package.go @@ -210,8 +210,8 @@ func (ic *ImportContext) Import(path string) (*types.Package, error) { return ic.Packages[a.ImportPath], nil } -// parseAllGoLinknames extracts all //go:linkname compiler directive from the sources. -func parseAllGoLinknames(s sources.Sources) ([]GoLinkname, error) { +// ParseAllGoLinknames extracts all //go:linkname compiler directive from the sources. +func ParseAllGoLinknames(s sources.Sources) ([]GoLinkname, error) { goLinknames := []GoLinkname{} var errs errorList.ErrorList for _, file := range s.Files { @@ -242,7 +242,8 @@ func Compile(srcs sources.Sources, importContext *ImportContext, minify bool) (_ err = bailout(fmt.Errorf("unexpected compiler panic while building package %q: %v", srcs.ImportPath, e)) }() - // TODO(grantnelson-wf): Clean up this function and fetch information + // TODO(grantnelson-wf): Clean up this function and fetch information. + // TODO(grantnelson-wf): Remove importContext since it is not used. importedPaths, importDecls := rootCtx.importDecls() From 6c48e0442f69d1e50738e70eb26fed619acd249a Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Thu, 27 Feb 2025 15:34:43 -0700 Subject: [PATCH 03/20] moved linkname to its own package --- compiler/compiler.go | 9 +++++---- compiler/compiler_test.go | 3 ++- compiler/{ => linkname}/linkname.go | 23 ++++++++++++++--------- compiler/{ => linkname}/linkname_test.go | 4 ++-- compiler/package.go | 14 +------------- compiler/sources/sources.go | 20 ++++++++++++++------ compiler/utils.go | 5 ----- 7 files changed, 38 insertions(+), 40 deletions(-) rename compiler/{ => linkname}/linkname.go (88%) rename compiler/{ => linkname}/linkname_test.go (98%) diff --git a/compiler/compiler.go b/compiler/compiler.go index 96ec390d8..e8264c946 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -18,6 +18,7 @@ import ( "time" "github.com/gopherjs/gopherjs/compiler/internal/dce" + "github.com/gopherjs/gopherjs/compiler/linkname" "github.com/gopherjs/gopherjs/compiler/prelude" "golang.org/x/tools/go/gcexportdata" ) @@ -59,7 +60,7 @@ type Archive struct { // Whether or not the package was compiled with minification enabled. Minified bool // A list of go:linkname directives encountered in the package. - GoLinknames []GoLinkname + GoLinknames []linkname.GoLinkname } func (a Archive) String() string { @@ -112,7 +113,7 @@ func WriteProgramCode(pkgs []*Archive, w *SourceMapFilter, goVersion string) err minify := mainPkg.Minified // Aggregate all go:linkname directives in the program together. - gls := goLinknameSet{} + gls := linkname.GoLinknameSet{} for _, pkg := range pkgs { gls.Add(pkg.GoLinknames) } @@ -164,7 +165,7 @@ func WriteProgramCode(pkgs []*Archive, w *SourceMapFilter, goVersion string) err return nil } -func WritePkgCode(pkg *Archive, dceSelection map[*Decl]struct{}, gls goLinknameSet, minify bool, w *SourceMapFilter) error { +func WritePkgCode(pkg *Archive, dceSelection map[*Decl]struct{}, gls linkname.GoLinknameSet, minify bool, w *SourceMapFilter) error { if w.MappingCallback != nil && pkg.FileSet != nil { w.fileSet = pkg.FileSet } @@ -264,7 +265,7 @@ type serializableArchive struct { IncJSCode []byte FileSet []byte Minified bool - GoLinknames []GoLinkname + GoLinknames []linkname.GoLinkname BuildTime time.Time } diff --git a/compiler/compiler_test.go b/compiler/compiler_test.go index 4c1dd4dba..0742cbaaa 100644 --- a/compiler/compiler_test.go +++ b/compiler/compiler_test.go @@ -12,6 +12,7 @@ import ( "golang.org/x/tools/go/packages" "github.com/gopherjs/gopherjs/compiler/internal/dce" + "github.com/gopherjs/gopherjs/compiler/linkname" "github.com/gopherjs/gopherjs/compiler/sources" "github.com/gopherjs/gopherjs/internal/srctesting" ) @@ -782,7 +783,7 @@ func renderPackage(t *testing.T, archive *Archive, minify bool) string { buf := &bytes.Buffer{} - if err := WritePkgCode(archive, selection, goLinknameSet{}, minify, &SourceMapFilter{Writer: buf}); err != nil { + if err := WritePkgCode(archive, selection, linkname.GoLinknameSet{}, minify, &SourceMapFilter{Writer: buf}); err != nil { t.Fatal(err) } diff --git a/compiler/linkname.go b/compiler/linkname/linkname.go similarity index 88% rename from compiler/linkname.go rename to compiler/linkname/linkname.go index af3ef6cc0..6c3a9623c 100644 --- a/compiler/linkname.go +++ b/compiler/linkname/linkname.go @@ -1,4 +1,4 @@ -package compiler +package linkname import ( "fmt" @@ -22,7 +22,7 @@ type GoLinkname struct { Reference symbol.Name } -// parseGoLinknames processed comments in a source file and extracts //go:linkname +// ParseGoLinknames processed comments in a source file and extracts //go:linkname // compiler directive from the comments. // // The following directive format is supported: @@ -38,7 +38,7 @@ type GoLinkname struct { // - The local function referenced by the directive must have no body (in other // words, it can only "import" an external function implementation into the // local scope). -func parseGoLinknames(fset *token.FileSet, pkgPath string, file *ast.File) ([]GoLinkname, error) { +func ParseGoLinknames(fset *token.FileSet, pkgPath string, file *ast.File) ([]GoLinkname, error) { var errs errorList.ErrorList = nil var directives []GoLinkname @@ -108,7 +108,7 @@ func parseGoLinknames(fset *token.FileSet, pkgPath string, file *ast.File) ([]Go for _, cg := range file.Comments { for _, c := range cg.List { if err := processComment(c); err != nil { - errs = append(errs, ErrorAt(err, fset, c.Pos())) + errs = append(errs, errorAt(err, fset, c.Pos())) } } } @@ -116,15 +116,20 @@ func parseGoLinknames(fset *token.FileSet, pkgPath string, file *ast.File) ([]Go return directives, errs.ErrOrNil() } -// goLinknameSet is a utility that enables quick lookup of whether a decl is +// errorAt annotates an error with a position in the source code. +func errorAt(err error, fset *token.FileSet, pos token.Pos) error { + return fmt.Errorf("%s: %w", fset.Position(pos), err) +} + +// GoLinknameSet is a utility that enables quick lookup of whether a decl is // affected by any go:linkname directive in the program. -type goLinknameSet struct { +type GoLinknameSet struct { byImplementation map[symbol.Name][]GoLinkname byReference map[symbol.Name]GoLinkname } // Add more GoLinkname directives into the set. -func (gls *goLinknameSet) Add(entries []GoLinkname) error { +func (gls *GoLinknameSet) Add(entries []GoLinkname) error { if gls.byImplementation == nil { gls.byImplementation = map[symbol.Name][]GoLinkname{} } @@ -144,7 +149,7 @@ func (gls *goLinknameSet) Add(entries []GoLinkname) error { // IsImplementation returns true if there is a directive referencing this symbol // as an implementation. -func (gls *goLinknameSet) IsImplementation(sym symbol.Name) bool { +func (gls *GoLinknameSet) IsImplementation(sym symbol.Name) bool { _, found := gls.byImplementation[sym] return found } @@ -152,7 +157,7 @@ func (gls *goLinknameSet) IsImplementation(sym symbol.Name) bool { // FindImplementation returns a symbol name, which provides the implementation // for the given symbol. The second value indicates whether the implementation // was found. -func (gls *goLinknameSet) FindImplementation(sym symbol.Name) (symbol.Name, bool) { +func (gls *GoLinknameSet) FindImplementation(sym symbol.Name) (symbol.Name, bool) { directive, found := gls.byReference[sym] return directive.Implementation, found } diff --git a/compiler/linkname_test.go b/compiler/linkname/linkname_test.go similarity index 98% rename from compiler/linkname_test.go rename to compiler/linkname/linkname_test.go index 7f46c6cfb..e2abc2825 100644 --- a/compiler/linkname_test.go +++ b/compiler/linkname/linkname_test.go @@ -1,4 +1,4 @@ -package compiler +package linkname import ( "go/ast" @@ -151,7 +151,7 @@ func TestParseGoLinknames(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { file, fset := parseSource(t, test.src) - directives, err := parseGoLinknames(fset, "testcase", file) + directives, err := ParseGoLinknames(fset, "testcase", file) if test.wantError != "" { if err == nil { diff --git a/compiler/package.go b/compiler/package.go index 239c9bf75..5345b5666 100644 --- a/compiler/package.go +++ b/compiler/package.go @@ -211,18 +211,6 @@ func (ic *ImportContext) Import(path string) (*types.Package, error) { return ic.Packages[a.ImportPath], nil } -// parseAllGoLinknames extracts all //go:linkname compiler directive from the sources. -func parseAllGoLinknames(s sources.Sources) ([]GoLinkname, error) { - goLinknames := []GoLinkname{} - var errs errorList.ErrorList - for _, file := range s.Files { - found, err := parseGoLinknames(s.FileSet, s.ImportPath, file) - errs = errs.Append(err) - goLinknames = append(goLinknames, found...) - } - return goLinknames, errs.ErrOrNil() -} - // Compile the provided Go sources as a single package. // // Import path must be the absolute import path for a package. Provided sources @@ -256,7 +244,7 @@ func Compile(srcs sources.Sources, importContext *ImportContext, minify bool) (_ importContext.Packages[srcs.ImportPath] = typesPkg // Extract all go:linkname compiler directives from the package source. - goLinknames, err := parseAllGoLinknames(srcs) + goLinknames, err := srcs.ParseGoLinknames() if err != nil { return nil, err } diff --git a/compiler/sources/sources.go b/compiler/sources/sources.go index fbcc98b95..053b5f5c7 100644 --- a/compiler/sources/sources.go +++ b/compiler/sources/sources.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/gopherjs/gopherjs/compiler/jsFile" + "github.com/gopherjs/gopherjs/compiler/linkname" "github.com/gopherjs/gopherjs/internal/errorList" "github.com/neelance/astrewrite" ) @@ -45,16 +46,11 @@ type Sources struct { // Note this function mutates the original slice. func (s Sources) Sort() Sources { sort.Slice(s.Files, func(i, j int) bool { - return s.nameOfFileAtIndex(i) > s.nameOfFileAtIndex(j) + return s.FileSet.File(s.Files[i].Pos()).Name() > s.FileSet.File(s.Files[j].Pos()).Name() }) return s } -// nameOfFileAtIndex gets the name of the Go source file at the given index. -func (s Sources) nameOfFileAtIndex(i int) string { - return s.FileSet.File(s.Files[i].Pos()).Name() -} - // Simplified returns a new sources instance with each Files entry processed by // astrewrite.Simplify. The JSFiles are copied unchanged. func (s Sources) Simplified(typesInfo *types.Info) Sources { @@ -114,6 +110,18 @@ func (s Sources) TypeCheck(importer types.Importer, sizes types.Sizes, tContext return typesInfo, typesPkg, nil } +// ParseGoLinknames extracts all //go:linkname compiler directive from the sources. +func (s Sources) ParseGoLinknames() ([]linkname.GoLinkname, error) { + goLinknames := []linkname.GoLinkname{} + var errs errorList.ErrorList + for _, file := range s.Files { + found, err := linkname.ParseGoLinknames(s.FileSet, s.ImportPath, file) + errs = errs.Append(err) + goLinknames = append(goLinknames, found...) + } + return goLinknames, errs.ErrOrNil() +} + // UnresolvedImports calculates the import paths of the package's dependencies // based on all the imports in the augmented Go AST files. // diff --git a/compiler/utils.go b/compiler/utils.go index 5d2cb4629..83b826ce2 100644 --- a/compiler/utils.go +++ b/compiler/utils.go @@ -933,11 +933,6 @@ func formatJSStructTagVal(jsTag string) string { return "." + jsTag } -// ErrorAt annotates an error with a position in the source code. -func ErrorAt(err error, fset *token.FileSet, pos token.Pos) error { - return fmt.Errorf("%s: %w", fset.Position(pos), err) -} - // FatalError is an error compiler panics with when it encountered a fatal error. // // FatalError implements io.Writer, which can be used to record any free-form From fb5299024e40ad087d91aaa04f68b2d79296c3b3 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Thu, 27 Feb 2025 16:12:38 -0700 Subject: [PATCH 04/20] Working on import context --- build/build.go | 9 ++++----- compiler/sources/sources.go | 20 ++++++-------------- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/build/build.go b/build/build.go index 3fb53b326..d2d963389 100644 --- a/build/build.go +++ b/build/build.go @@ -1107,7 +1107,7 @@ func (s *Session) loadPackages(pkg *PackageData) (*sources.Sources, error) { } func (s *Session) prepareSources() error { - for importPath, srcs := range s.sources { + for _, srcs := range s.sources { tContext := types.NewContext() typesInfo, typesPkg, err := srcs.TypeCheck(importContext, sizes32, tContext) @@ -1117,16 +1117,15 @@ func (s *Session) prepareSources() error { if genErr := typeparams.RequiresGenericsSupport(typesInfo); genErr != nil && !experiments.Env.Generics { return fmt.Errorf("package %s requires generics support (https://github.com/gopherjs/gopherjs/issues/1013): %w", srcs.ImportPath, genErr) } - s.Types[importPath] = typesPkg + s.Types[srcs.ImportPath] = typesPkg // Extract all go:linkname compiler directives from the package source. - goLinknames, err := parseAllGoLinknames(srcs) + goLinknames, err := srcs.ParseGoLinknames() if err != nil { return err } - srcs = srcs.Simplified(typesInfo) - s.sources[importPath] = srcs + srcs.Simplify(typesInfo) rootCtx := newRootCtx(tContext, srcs, typesInfo, typesPkg, importContext.isBlocking, minify) diff --git a/compiler/sources/sources.go b/compiler/sources/sources.go index 3e9ae9e0f..2130a4efa 100644 --- a/compiler/sources/sources.go +++ b/compiler/sources/sources.go @@ -43,7 +43,7 @@ type Sources struct { // Sort the Go files slice by the original source name to ensure consistent order // of processing. This is required for reproducible JavaScript output. // -// Note this function mutates the original slice. +// Note this function mutates the original Files slice. func (s Sources) Sort() Sources { sort.Slice(s.Files, func(i, j int) bool { return s.FileSet.File(s.Files[i].Pos()).Name() > s.FileSet.File(s.Files[j].Pos()).Name() @@ -51,21 +51,13 @@ func (s Sources) Sort() Sources { return s } -// Simplified returns a new sources instance with each Files entry processed by -// astrewrite.Simplify. The JSFiles are copied unchanged. -func (s Sources) Simplified(typesInfo *types.Info) Sources { - simplified := Sources{ - ImportPath: s.ImportPath, - Dir: s.Dir, - Files: make([]*ast.File, len(s.Files)), - FileSet: s.FileSet, - JSFiles: s.JSFiles, - } +// Simplify processed each Files entry with astrewrite.Simplify. +// +// Note this function mutates the original Files slice. +func (s Sources) Simplify(typesInfo *types.Info) { for i, file := range s.Files { - simplified.Files[i] = astrewrite.Simplify(file, typesInfo, false) + s.Files[i] = astrewrite.Simplify(file, typesInfo, false) } - simplified.Sort() - return simplified } // TypeCheck the sources. Returns information about declared package types and From cbf9b5869cfc80e3a9971732a9b2f834f5f65753 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Fri, 28 Feb 2025 10:34:46 -0700 Subject: [PATCH 05/20] Working on import context --- build/build.go | 99 +++++++++++++++++++------------------ compiler/package.go | 9 ++-- compiler/sources/sources.go | 47 +++++++++++++----- 3 files changed, 92 insertions(+), 63 deletions(-) diff --git a/build/build.go b/build/build.go index d2d963389..105f5b604 100644 --- a/build/build.go +++ b/build/build.go @@ -768,8 +768,8 @@ type Session struct { // up to date with input sources and dependencies. In the -w ("watch") mode // must be cleared upon entering watching. UpToDateArchives map[string]*compiler.Archive - Types map[string]*types.Package - Watcher *fsnotify.Watcher + + Watcher *fsnotify.Watcher } // NewSession creates a new GopherJS build session. @@ -790,7 +790,6 @@ func NewSession(options *Options) (*Session, error) { return nil, err } - s.Types = make(map[string]*types.Package) if options.Watch { if out, err := exec.Command("ulimit", "-n").Output(); err == nil { if n, err := strconv.Atoi(strings.TrimSpace(string(out))); err == nil && n < 1024 { @@ -908,7 +907,7 @@ func (s *Session) BuildFiles(filenames []string, pkgObj string, cwd string) erro if err != nil { return err } - if s.Types["main"].Name() != "main" { + if s.sources["main"].Package.Name() != "main" { return fmt.Errorf("cannot build/run non-main package") } return s.WriteCommandPackage(archive, pkgObj) @@ -932,11 +931,15 @@ func (s *Session) BuildProject(pkg *PackageData) (*compiler.Archive, error) { return nil, err } - // Prepare and analyze the source code for compiling. - if err := s.prepareSources(); err != nil { + // Prepare and analyze the source code. + // This will be performed recursively for all dependencies. + tContext := types.NewContext() + if err = s.prepareSources(srcs, tContext); err != nil { return nil, err } + // TODO(grantnelson-wf): Handle newRootCtx changes + // Compile the project into Archives containing the generated JS. return s.compilePackages(srcs) } @@ -1106,60 +1109,61 @@ func (s *Session) loadPackages(pkg *PackageData) (*sources.Sources, error) { return srcs, nil } -func (s *Session) prepareSources() error { - for _, srcs := range s.sources { +type importContext struct { + s *Session + dir string + tContext *types.Context +} - tContext := types.NewContext() - typesInfo, typesPkg, err := srcs.TypeCheck(importContext, sizes32, tContext) - if err != nil { - return err - } - if genErr := typeparams.RequiresGenericsSupport(typesInfo); genErr != nil && !experiments.Env.Generics { - return fmt.Errorf("package %s requires generics support (https://github.com/gopherjs/gopherjs/issues/1013): %w", srcs.ImportPath, genErr) - } - s.Types[srcs.ImportPath] = typesPkg +func (ic *importContext) Import(path string) (*types.Package, error) { - // Extract all go:linkname compiler directives from the package source. - goLinknames, err := srcs.ParseGoLinknames() - if err != nil { - return err - } +} - srcs.Simplify(typesInfo) +func (s *Session) prepareSources(srcs *sources.Sources, tContext *types.Context) error { + ic := &importContext{ + s: s, + dir: srcs.Dir, + tContext: tContext, + } + + if err := srcs.TypeCheck(ic, sizes32, tContext); err != nil { + return err + } - rootCtx := newRootCtx(tContext, srcs, typesInfo, typesPkg, importContext.isBlocking, minify) + if genErr := typeparams.RequiresGenericsSupport(srcs.TypeInfo); genErr != nil && !experiments.Env.Generics { + return fmt.Errorf("package %s requires generics support (https://github.com/gopherjs/gopherjs/issues/1013): %w", srcs.ImportPath, genErr) + } + // Extract all go:linkname compiler directives from the package source. + if err := srcs.ParseGoLinknames(); err != nil { + return err } + + srcs.Simplify() return nil } -func (s *Session) compilePackages(srcs *sources.Sources) (*compiler.Archive, error) { - if archive, ok := s.UpToDateArchives[srcs.ImportPath]; ok { - return archive, nil - } +func (s *Session) compilePackages(rootSrcs *sources.Sources) (*compiler.Archive, error) { + for _, srcs := range s.sources { + archive, err := compiler.Compile(*srcs, s.options.Minify) + if err != nil { + return nil, err + } - importContext := &compiler.ImportContext{ - Packages: s.Types, - ImportArchive: s.ImportResolverFor(srcs.Dir), - } - archive, err := compiler.Compile(*srcs, importContext, s.options.Minify) - if err != nil { - return nil, err - } + for _, jsFile := range srcs.JSFiles { + archive.IncJSCode = append(archive.IncJSCode, []byte("\t(function() {\n")...) + archive.IncJSCode = append(archive.IncJSCode, jsFile.Content...) + archive.IncJSCode = append(archive.IncJSCode, []byte("\n\t}).call($global);\n")...) + } - for _, jsFile := range srcs.JSFiles { - archive.IncJSCode = append(archive.IncJSCode, []byte("\t(function() {\n")...) - archive.IncJSCode = append(archive.IncJSCode, jsFile.Content...) - archive.IncJSCode = append(archive.IncJSCode, []byte("\n\t}).call($global);\n")...) - } + if s.options.Verbose { + fmt.Println(srcs.ImportPath) + } - if s.options.Verbose { - fmt.Println(srcs.ImportPath) + s.UpToDateArchives[srcs.ImportPath] = archive } - s.UpToDateArchives[srcs.ImportPath] = archive - - return archive, nil + return s.UpToDateArchives[rootSrcs.ImportPath], nil } func (s *Session) getImportPath(path, srcDir string) (string, error) { @@ -1288,8 +1292,9 @@ func hasGopathPrefix(file, gopath string) (hasGopathPrefix bool, prefixLen int) func (s *Session) WaitForChange() { // Will need to re-validate up-to-dateness of all archives, so flush them from // memory. + s.importPaths = map[string]map[string]string{} + s.sources = map[string]*sources.Sources{} s.UpToDateArchives = map[string]*compiler.Archive{} - s.Types = map[string]*types.Package{} s.options.PrintSuccess("watching for changes...\n") for { diff --git a/compiler/package.go b/compiler/package.go index a0cdf742c..8c7fcf4a8 100644 --- a/compiler/package.go +++ b/compiler/package.go @@ -212,9 +212,8 @@ func (ic *ImportContext) Import(path string) (*types.Package, error) { // Compile the provided Go sources as a single package. // -// Import path must be the absolute import path for a package. Provided sources -// are always sorted by name to ensure reproducible JavaScript output. -func Compile(srcs sources.Sources, importContext *ImportContext, minify bool) (_ *Archive, err error) { +// Provided sources must be sorted by name to ensure reproducible JavaScript output. +func Compile(srcs sources.Sources, minify bool) (_ *Archive, err error) { defer func() { e := recover() if e == nil { @@ -230,8 +229,8 @@ func Compile(srcs sources.Sources, importContext *ImportContext, minify bool) (_ err = bailout(fmt.Errorf("unexpected compiler panic while building package %q: %v", srcs.ImportPath, e)) }() - // TODO(grantnelson-wf): Clean up this function and fetch information. - // TODO(grantnelson-wf): Remove importContext since it is not used. + // TODO(grantnelson-wf): Move rootCtx to a separate function to perform cross package analysis. + rootCtx := newRootCtx(tContext, srcs, typesInfo, typesPkg, importContext.isBlocking, minify) importedPaths, importDecls := rootCtx.importDecls() diff --git a/compiler/sources/sources.go b/compiler/sources/sources.go index 2130a4efa..171dc4e8f 100644 --- a/compiler/sources/sources.go +++ b/compiler/sources/sources.go @@ -38,31 +38,46 @@ type Sources struct { // JSFiles is the JavaScript files that are part of the package. JSFiles []jsFile.JSFile + + // TypeInfo is the type information this package. + // This is nil until TypeCheck is called. + TypeInfo *types.Info + + // Package is the type-checked package. + // This is nil until TypeCheck is called. + Package *types.Package + + // GoLinknames is the set of Go linknames for this package. + // This is nil until ParseGoLinknames is called. + GoLinknames []linkname.GoLinkname } // Sort the Go files slice by the original source name to ensure consistent order // of processing. This is required for reproducible JavaScript output. // // Note this function mutates the original Files slice. -func (s Sources) Sort() Sources { +func (s Sources) Sort() { sort.Slice(s.Files, func(i, j int) bool { return s.FileSet.File(s.Files[i].Pos()).Name() > s.FileSet.File(s.Files[j].Pos()).Name() }) - return s } // Simplify processed each Files entry with astrewrite.Simplify. // // Note this function mutates the original Files slice. -func (s Sources) Simplify(typesInfo *types.Info) { +// This must be called after TypeCheck. +func (s Sources) Simplify() { for i, file := range s.Files { - s.Files[i] = astrewrite.Simplify(file, typesInfo, false) + s.Files[i] = astrewrite.Simplify(file, s.TypeInfo, false) } } // TypeCheck the sources. Returns information about declared package types and // type information for the supplied AST. -func (s Sources) TypeCheck(importer types.Importer, sizes types.Sizes, tContext *types.Context) (*types.Info, *types.Package, error) { +// +// This will set the Info and Package fields on the Sources struct. +// This must be called prior to Simplify. +func (s Sources) TypeCheck(importer types.Importer, sizes types.Sizes, tContext *types.Context) error { const errLimit = 10 // Max number of type checking errors to return. typesInfo := &types.Info{ @@ -90,21 +105,27 @@ func (s Sources) TypeCheck(importer types.Importer, sizes types.Sizes, tContext // are not meaningful and would be resolved by fixing imports. Return them // separately, if any. https://github.com/gopherjs/gopherjs/issues/119. if ecImporter.Errors.ErrOrNil() != nil { - return nil, nil, ecImporter.Errors.Trim(errLimit).ErrOrNil() + return ecImporter.Errors.Trim(errLimit).ErrOrNil() } // Return any other type errors. if typeErrs.ErrOrNil() != nil { - return nil, nil, typeErrs.Trim(errLimit).ErrOrNil() + return typeErrs.Trim(errLimit).ErrOrNil() } // Any general errors that may have occurred during type checking. if err != nil { - return nil, nil, err + return err } - return typesInfo, typesPkg, nil + + s.TypeInfo = typesInfo + s.Package = typesPkg + return nil } // ParseGoLinknames extracts all //go:linkname compiler directive from the sources. -func (s Sources) ParseGoLinknames() ([]linkname.GoLinkname, error) { +// +// This will set the GoLinknames field on the Sources struct. +// This must be called prior to Simplify. +func (s Sources) ParseGoLinknames() error { goLinknames := []linkname.GoLinkname{} var errs errorList.ErrorList for _, file := range s.Files { @@ -112,7 +133,11 @@ func (s Sources) ParseGoLinknames() ([]linkname.GoLinkname, error) { errs = errs.Append(err) goLinknames = append(goLinknames, found...) } - return goLinknames, errs.ErrOrNil() + if err := errs.ErrOrNil(); err != nil { + return err + } + s.GoLinknames = goLinknames + return nil } // UnresolvedImports calculates the import paths of the package's dependencies From 36c8c11c225ad5e1abad8b5f99c3a25a20342745 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Fri, 28 Feb 2025 12:27:03 -0700 Subject: [PATCH 06/20] Working on package analysis --- build/build.go | 61 +++++-------- compiler/internal/analysis/info.go | 56 ++++++++---- compiler/internal/analysis/info_test.go | 27 +++--- compiler/package.go | 94 ++++++++------------- compiler/sources/errorCollectingImporter.go | 10 ++- compiler/sources/sources.go | 39 ++++++++- 6 files changed, 152 insertions(+), 135 deletions(-) diff --git a/build/build.go b/build/build.go index 105f5b604..32fe5c78a 100644 --- a/build/build.go +++ b/build/build.go @@ -27,11 +27,9 @@ import ( "github.com/fsnotify/fsnotify" "github.com/gopherjs/gopherjs/compiler" "github.com/gopherjs/gopherjs/compiler/astutil" - "github.com/gopherjs/gopherjs/compiler/internal/typeparams" "github.com/gopherjs/gopherjs/compiler/jsFile" "github.com/gopherjs/gopherjs/compiler/sources" "github.com/gopherjs/gopherjs/internal/errorList" - "github.com/gopherjs/gopherjs/internal/experiments" "github.com/gopherjs/gopherjs/internal/testmain" log "github.com/sirupsen/logrus" @@ -938,10 +936,11 @@ func (s *Session) BuildProject(pkg *PackageData) (*compiler.Archive, error) { return nil, err } - // TODO(grantnelson-wf): Handle newRootCtx changes + // Propagate the analysis information to all packages. + compiler.PropagateAnalysis(s.sources) // Compile the project into Archives containing the generated JS. - return s.compilePackages(srcs) + return s.compilePackages(srcs, tContext) } func (s *Session) loadTestPackage(pkg *PackageData) (*sources.Sources, error) { @@ -1109,43 +1108,32 @@ func (s *Session) loadPackages(pkg *PackageData) (*sources.Sources, error) { return srcs, nil } -type importContext struct { - s *Session - dir string - tContext *types.Context -} - -func (ic *importContext) Import(path string) (*types.Package, error) { - -} - func (s *Session) prepareSources(srcs *sources.Sources, tContext *types.Context) error { - ic := &importContext{ - s: s, - dir: srcs.Dir, - tContext: tContext, - } - - if err := srcs.TypeCheck(ic, sizes32, tContext); err != nil { - return err - } + importer := func(path string) (*sources.Sources, error) { + importPath, err := s.getImportPath(path, srcs.Dir) + if err != nil { + return nil, err + } - if genErr := typeparams.RequiresGenericsSupport(srcs.TypeInfo); genErr != nil && !experiments.Env.Generics { - return fmt.Errorf("package %s requires generics support (https://github.com/gopherjs/gopherjs/issues/1013): %w", srcs.ImportPath, genErr) - } + if srcs, ok := s.sources[importPath]; ok { + if srcs.Package == nil { + err = s.prepareSources(srcs, tContext) + if err != nil { + return nil, err + } + } + return srcs, nil + } - // Extract all go:linkname compiler directives from the package source. - if err := srcs.ParseGoLinknames(); err != nil { - return err + return nil, fmt.Errorf(`sources for %q not found`, importPath) } - srcs.Simplify() - return nil + return compiler.PrepareSources(srcs, importer, tContext) } -func (s *Session) compilePackages(rootSrcs *sources.Sources) (*compiler.Archive, error) { +func (s *Session) compilePackages(rootSrcs *sources.Sources, tContext *types.Context) (*compiler.Archive, error) { for _, srcs := range s.sources { - archive, err := compiler.Compile(*srcs, s.options.Minify) + archive, err := compiler.Compile(*srcs, tContext, s.options.Minify) if err != nil { return nil, err } @@ -1199,12 +1187,7 @@ func (s *Session) ImportResolverFor(srcDir string) func(string) (*compiler.Archi return archive, nil } - // The archive hasn't been compiled yet so compile it with the sources. - if srcs, ok := s.sources[importPath]; ok { - return s.compilePackages(srcs) - } - - return nil, fmt.Errorf(`sources for %q not found`, importPath) + return nil, fmt.Errorf(`archive for %q not found`, importPath) } } diff --git a/compiler/internal/analysis/info.go b/compiler/internal/analysis/info.go index 803952b24..f5bebc91d 100644 --- a/compiler/internal/analysis/info.go +++ b/compiler/internal/analysis/info.go @@ -58,10 +58,12 @@ type Info struct { funcLitInfos map[*ast.FuncLit][]*FuncInfo InitFuncInfo *FuncInfo // Context for package variable initialization. - isImportedBlocking func(typeparams.Instance) bool // For functions from other packages. - allInfos []*FuncInfo + infoImporter InfoImporter // For functions from other packages. + allInfos []*FuncInfo } +type InfoImporter func(path string) (*Info, error) + func (info *Info) newFuncInfo(n ast.Node, obj types.Object, typeArgs typesutil.TypeList, resolver *typeparams.Resolver) *FuncInfo { funcInfo := &FuncInfo{ pkgInfo: info, @@ -132,11 +134,16 @@ func (info *Info) newFuncInfoInstances(fd *ast.FuncDecl) []*FuncInfo { } // IsBlocking returns true if the function may contain blocking calls or operations. -// If inst is from a different package, this will use the isImportedBlocking +// If inst is from a different package, this will use the getImportInfo function // to lookup the information from the other package. func (info *Info) IsBlocking(inst typeparams.Instance) bool { if inst.Object.Pkg() != info.Pkg { - return info.isImportedBlocking(inst) + path := inst.Object.Pkg().Path() + otherInfo, err := info.infoImporter(path) + if err != nil { + panic(fmt.Errorf(`failed to get info for package %q: %v`, path, err)) + } + return otherInfo.IsBlocking(inst) } if funInfo := info.FuncInfo(inst); funInfo != nil { return funInfo.IsBlocking() @@ -174,16 +181,21 @@ func (info *Info) VarsWithInitializers() map[*types.Var]bool { return result } -func AnalyzePkg(files []*ast.File, fileSet *token.FileSet, typesInfo *types.Info, typeCtx *types.Context, typesPkg *types.Package, instanceSets *typeparams.PackageInstanceSets, isBlocking func(typeparams.Instance) bool) *Info { +// AnalyzePkg analyzes the given package for blocking calls, defers, etc. +// +// Note that at the end of this call the analysis information +// has NOT been propagated across packages yet. Once all the packages +// have been analyzed, call PropagateAnalysis to propagate the information. +func AnalyzePkg(files []*ast.File, fileSet *token.FileSet, typesInfo *types.Info, typeCtx *types.Context, typesPkg *types.Package, instanceSets *typeparams.PackageInstanceSets, infoImporter InfoImporter) *Info { info := &Info{ - Info: typesInfo, - Pkg: typesPkg, - typeCtx: typeCtx, - instanceSets: instanceSets, - HasPointer: make(map[*types.Var]bool), - isImportedBlocking: isBlocking, - funcInstInfos: new(typeparams.InstanceMap[*FuncInfo]), - funcLitInfos: make(map[*ast.FuncLit][]*FuncInfo), + Info: typesInfo, + Pkg: typesPkg, + typeCtx: typeCtx, + instanceSets: instanceSets, + HasPointer: make(map[*types.Var]bool), + infoImporter: infoImporter, + funcInstInfos: new(typeparams.InstanceMap[*FuncInfo]), + funcLitInfos: make(map[*ast.FuncLit][]*FuncInfo), } info.InitFuncInfo = info.newFuncInfo(nil, nil, nil, nil) @@ -193,13 +205,25 @@ func AnalyzePkg(files []*ast.File, fileSet *token.FileSet, typesInfo *types.Info ast.Walk(info.InitFuncInfo, file) } + return info +} + +// PropagateAnalysis will propagate analysis information across package +// boundaries to finish the analysis of a whole project. +func PropagateAnalysis(allInfo []*Info) { done := false for !done { - done = info.propagateFunctionBlocking() + done = true + for _, info := range allInfo { + if !info.propagateFunctionBlocking() { + done = false + } + } } - info.propagateControlStatementBlocking() - return info + for _, info := range allInfo { + info.propagateControlStatementBlocking() + } } // propagateFunctionBlocking propagates information about blocking calls diff --git a/compiler/internal/analysis/info_test.go b/compiler/internal/analysis/info_test.go index 957e346d9..0e83a04bc 100644 --- a/compiler/internal/analysis/info_test.go +++ b/compiler/internal/analysis/info_test.go @@ -1,6 +1,7 @@ package analysis import ( + "fmt" "go/ast" "go/types" "sort" @@ -1639,11 +1640,10 @@ func newBlockingTest(t *testing.T, src string) *blockingTest { testInfo, testPkg := f.Check(`pkg/test`, file) tc.Scan(testPkg, file) - isImportBlocking := func(i typeparams.Instance) bool { - t.Fatalf(`isImportBlocking should not be called in this test, called with %v`, i) - return true + getImportInfo := func(path string) (*Info, error) { + return nil, fmt.Errorf(`getImportInfo should not be called in this test, called with %v`, path) } - pkgInfo := AnalyzePkg([]*ast.File{file}, f.FileSet, testInfo, types.NewContext(), testPkg, tc.Instances, isImportBlocking) + pkgInfo := AnalyzePkg([]*ast.File{file}, f.FileSet, testInfo, types.NewContext(), testPkg, tc.Instances, getImportInfo) return &blockingTest{ f: f, @@ -1660,13 +1660,12 @@ func newBlockingTestWithOtherPackage(t *testing.T, testSrc string, otherSrc stri Instances: &typeparams.PackageInstanceSets{}, } - pkgInfo := map[*types.Package]*Info{} - isImportBlocking := func(i typeparams.Instance) bool { - if info, ok := pkgInfo[i.Object.Pkg()]; ok { - return info.IsBlocking(i) + pkgInfo := map[string]*Info{} + getImportInfo := func(path string) (*Info, error) { + if info, ok := pkgInfo[path]; ok { + return info, nil } - t.Fatalf(`unexpected package in isImportBlocking for %v`, i) - return true + return nil, fmt.Errorf(`unexpected package in getImportInfo for %v`, path) } otherFile := f.Parse(`other.go`, otherSrc) @@ -1677,11 +1676,11 @@ func newBlockingTestWithOtherPackage(t *testing.T, testSrc string, otherSrc stri _, testPkg := f.Check(`pkg/test`, testFile) tc.Scan(testPkg, testFile) - otherPkgInfo := AnalyzePkg([]*ast.File{otherFile}, f.FileSet, f.Info, types.NewContext(), otherPkg, tc.Instances, isImportBlocking) - pkgInfo[otherPkg] = otherPkgInfo + otherPkgInfo := AnalyzePkg([]*ast.File{otherFile}, f.FileSet, f.Info, types.NewContext(), otherPkg, tc.Instances, getImportInfo) + pkgInfo[otherPkg.Path()] = otherPkgInfo - testPkgInfo := AnalyzePkg([]*ast.File{testFile}, f.FileSet, f.Info, types.NewContext(), testPkg, tc.Instances, isImportBlocking) - pkgInfo[testPkg] = testPkgInfo + testPkgInfo := AnalyzePkg([]*ast.File{testFile}, f.FileSet, f.Info, types.NewContext(), testPkg, tc.Instances, getImportInfo) + pkgInfo[testPkg.Path()] = testPkgInfo return &blockingTest{ f: f, diff --git a/compiler/package.go b/compiler/package.go index 8c7fcf4a8..07ec412f4 100644 --- a/compiler/package.go +++ b/compiler/package.go @@ -15,6 +15,7 @@ import ( "github.com/gopherjs/gopherjs/compiler/sources" "github.com/gopherjs/gopherjs/compiler/typesutil" "github.com/gopherjs/gopherjs/internal/errorList" + "github.com/gopherjs/gopherjs/internal/experiments" ) // pkgContext maintains compiler context for a specific package. @@ -116,18 +117,11 @@ type funcContext struct { funcLitCounter int } -func newRootCtx(tContext *types.Context, srcs sources.Sources, typesInfo *types.Info, typesPkg *types.Package, isBlocking func(typeparams.Instance) bool, minify bool) *funcContext { - tc := typeparams.Collector{ - TContext: tContext, - Info: typesInfo, - Instances: &typeparams.PackageInstanceSets{}, - } - tc.Scan(typesPkg, srcs.Files...) - pkgInfo := analysis.AnalyzePkg(srcs.Files, srcs.FileSet, typesInfo, tContext, typesPkg, tc.Instances, isBlocking) +func newRootCtx(tContext *types.Context, srcs sources.Sources, minify bool) *funcContext { funcCtx := &funcContext{ - FuncInfo: pkgInfo.InitFuncInfo, + FuncInfo: srcs.TypeInfo.InitFuncInfo, pkgCtx: &pkgContext{ - Info: pkgInfo, + Info: srcs.TypeInfo, additionalSelections: make(map[*ast.SelectorExpr]typesutil.Selection), typesCtx: tContext, @@ -137,7 +131,7 @@ func newRootCtx(tContext *types.Context, srcs sources.Sources, typesInfo *types. indentation: 1, minify: minify, fileSet: srcs.FileSet, - instanceSet: tc.Instances, + instanceSet: srcs.Instances, }, allVars: make(map[string]int), flowDatas: map[*types.Label]*flowData{nil: {}}, @@ -157,63 +151,46 @@ type flowData struct { endCase int } -// ImportContext provides access to information about imported packages. -type ImportContext struct { - // Mapping for an absolute import path to the package type information. - Packages map[string]*types.Package - // ImportArchive returns a previously compiled Archive for a dependency - // package. If the Import() call was successful, the corresponding entry - // must be added to the Packages map. - ImportArchive func(importPath string) (*Archive, error) -} - -// isBlocking returns true if an _imported_ function is blocking. It will panic -// if the function decl is not found in the imported package or the package -// hasn't been compiled yet. +// PrepareSources recursively processes the provided sources and +// prepares them for compilation by determining the type information, +// go linknames, and simplifying the AST. // -// Note: see analysis.FuncInfo.Blocking if you need to determine if a function -// in the _current_ package is blocking. Usually available via functionContext -// object. -func (ic *ImportContext) isBlocking(inst typeparams.Instance) bool { - f, ok := inst.Object.(*types.Func) - if !ok { - panic(bailout(fmt.Errorf("can't determine if instance %v is blocking: instance isn't for a function object", inst))) +// Note that at the end of this call the analysis information +// has NOT been propagated across packages yet. +func PrepareSources(srcs *sources.Sources, importer sources.SourcesImporter, tContext *types.Context) error { + if err := srcs.TypeCheck(importer, sizes32, tContext); err != nil { + return err } - archive, err := ic.ImportArchive(f.Pkg().Path()) - if err != nil { - panic(err) + if genErr := typeparams.RequiresGenericsSupport(srcs.TypeInfo.Info); genErr != nil && !experiments.Env.Generics { + return fmt.Errorf("package %s requires generics support (https://github.com/gopherjs/gopherjs/issues/1013): %w", srcs.ImportPath, genErr) } - fullName := funcDeclFullName(inst) - for _, d := range archive.Declarations { - if d.FullName == fullName { - return d.Blocking - } + // Extract all go:linkname compiler directives from the package source. + if err := srcs.ParseGoLinknames(); err != nil { + return err } - panic(bailout(fmt.Errorf("can't determine if function %s is blocking: decl not found in package archive", fullName))) -} -// Import implements go/types.Importer interface for ImportContext. -func (ic *ImportContext) Import(path string) (*types.Package, error) { - if path == "unsafe" { - return types.Unsafe, nil - } + srcs.Simplify() + return nil +} - // By importing the archive, the package will compile if it hasn't been - // compiled yet and the package will be added to the Packages map. - a, err := ic.ImportArchive(path) - if err != nil { - return nil, err +// PropagateAnalysis the analysis information to all packages. +// +// The map of sources is keyed by the resolved import path of the package, +// however that key is not used. +func PropagateAnalysis(allSources map[string]*sources.Sources) { + allInfo := make([]*analysis.Info, 0, len(allSources)) + for _, src := range allSources { + allInfo = append(allInfo, src.TypeInfo) } - - return ic.Packages[a.ImportPath], nil + analysis.PropagateAnalysis(allInfo) } // Compile the provided Go sources as a single package. // // Provided sources must be sorted by name to ensure reproducible JavaScript output. -func Compile(srcs sources.Sources, minify bool) (_ *Archive, err error) { +func Compile(srcs sources.Sources, tContext *types.Context, minify bool) (_ *Archive, err error) { defer func() { e := recover() if e == nil { @@ -229,8 +206,7 @@ func Compile(srcs sources.Sources, minify bool) (_ *Archive, err error) { err = bailout(fmt.Errorf("unexpected compiler panic while building package %q: %v", srcs.ImportPath, e)) }() - // TODO(grantnelson-wf): Move rootCtx to a separate function to perform cross package analysis. - rootCtx := newRootCtx(tContext, srcs, typesInfo, typesPkg, importContext.isBlocking, minify) + rootCtx := newRootCtx(tContext, srcs, minify) importedPaths, importDecls := rootCtx.importDecls() @@ -274,13 +250,13 @@ func Compile(srcs sources.Sources, minify bool) (_ *Archive, err error) { return &Archive{ ImportPath: srcs.ImportPath, - Name: typesPkg.Name(), + Name: srcs.Package.Name(), Imports: importedPaths, - Package: typesPkg, + Package: srcs.Package, Declarations: allDecls, FileSet: srcs.FileSet, Minified: minify, - GoLinknames: goLinknames, + GoLinknames: srcs.GoLinknames, }, nil } diff --git a/compiler/sources/errorCollectingImporter.go b/compiler/sources/errorCollectingImporter.go index 73bd3c65d..a3bdb246c 100644 --- a/compiler/sources/errorCollectingImporter.go +++ b/compiler/sources/errorCollectingImporter.go @@ -9,15 +9,19 @@ import ( // errorCollectingImporter implements go/types.Importer interface and // wraps it to collect import errors. type errorCollectingImporter struct { - Importer types.Importer + Importer SourcesImporter Errors errorList.ErrorList } func (ei *errorCollectingImporter) Import(path string) (*types.Package, error) { - pkg, err := ei.Importer.Import(path) + if path == "unsafe" { + return types.Unsafe, nil + } + + srcs, err := ei.Importer(path) if err != nil { ei.Errors = ei.Errors.AppendDistinct(err) return nil, err } - return pkg, nil + return srcs.Package, nil } diff --git a/compiler/sources/sources.go b/compiler/sources/sources.go index 171dc4e8f..a989bcc59 100644 --- a/compiler/sources/sources.go +++ b/compiler/sources/sources.go @@ -7,6 +7,8 @@ import ( "sort" "strings" + "github.com/gopherjs/gopherjs/compiler/internal/analysis" + "github.com/gopherjs/gopherjs/compiler/internal/typeparams" "github.com/gopherjs/gopherjs/compiler/jsFile" "github.com/gopherjs/gopherjs/compiler/linkname" "github.com/gopherjs/gopherjs/internal/errorList" @@ -41,7 +43,11 @@ type Sources struct { // TypeInfo is the type information this package. // This is nil until TypeCheck is called. - TypeInfo *types.Info + TypeInfo *analysis.Info + + // Instances is the type parameters instances for this package. + // This is nil until TypeCheck is called. + Instances *typeparams.PackageInstanceSets // Package is the type-checked package. // This is nil until TypeCheck is called. @@ -52,6 +58,8 @@ type Sources struct { GoLinknames []linkname.GoLinkname } +type SourcesImporter func(path string) (*Sources, error) + // Sort the Go files slice by the original source name to ensure consistent order // of processing. This is required for reproducible JavaScript output. // @@ -68,7 +76,7 @@ func (s Sources) Sort() { // This must be called after TypeCheck. func (s Sources) Simplify() { for i, file := range s.Files { - s.Files[i] = astrewrite.Simplify(file, s.TypeInfo, false) + s.Files[i] = astrewrite.Simplify(file, s.TypeInfo.Info, false) } } @@ -77,7 +85,9 @@ func (s Sources) Simplify() { // // This will set the Info and Package fields on the Sources struct. // This must be called prior to Simplify. -func (s Sources) TypeCheck(importer types.Importer, sizes types.Sizes, tContext *types.Context) error { +// Note that at the end of this call the analysis information +// has NOT been propagated across packages yet. +func (s Sources) TypeCheck(importer SourcesImporter, sizes types.Sizes, tContext *types.Context) error { const errLimit = 10 // Max number of type checking errors to return. typesInfo := &types.Info{ @@ -116,11 +126,32 @@ func (s Sources) TypeCheck(importer types.Importer, sizes types.Sizes, tContext return err } - s.TypeInfo = typesInfo + tc := typeparams.Collector{ + TContext: tContext, + Info: typesInfo, + Instances: &typeparams.PackageInstanceSets{}, + } + tc.Scan(typesPkg, s.Files...) + + infoImporter := getInfoImporter(importer) + anaInfo := analysis.AnalyzePkg(s.Files, s.FileSet, typesInfo, tContext, typesPkg, tc.Instances, infoImporter) + + s.TypeInfo = anaInfo + s.Instances = tc.Instances s.Package = typesPkg return nil } +func getInfoImporter(importer SourcesImporter) analysis.InfoImporter { + return func(path string) (*analysis.Info, error) { + srcs, err := importer(path) + if err != nil { + return nil, err + } + return srcs.TypeInfo, nil + } +} + // ParseGoLinknames extracts all //go:linkname compiler directive from the sources. // // This will set the GoLinknames field on the Sources struct. From cd071c15cdf479bc315d2c13f4971cf3227a8c67 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Fri, 28 Feb 2025 16:08:07 -0700 Subject: [PATCH 07/20] Moving propagate --- build/build.go | 17 ++- compiler/compiler_test.go | 4 + compiler/package.go | 42 ++++---- compiler/sources/errorCollectingImporter.go | 10 +- compiler/sources/sources.go | 111 ++++++++++++++------ 5 files changed, 124 insertions(+), 60 deletions(-) diff --git a/build/build.go b/build/build.go index 32fe5c78a..6783c09c5 100644 --- a/build/build.go +++ b/build/build.go @@ -1093,7 +1093,6 @@ func (s *Session) loadPackages(pkg *PackageData) (*sources.Sources, error) { FileSet: fileSet, JSFiles: append(pkg.JSFiles, overlayJsFiles...), } - srcs.Sort() s.sources[pkg.ImportPath] = srcs // Import dependencies from the augmented files, @@ -1128,7 +1127,7 @@ func (s *Session) prepareSources(srcs *sources.Sources, tContext *types.Context) return nil, fmt.Errorf(`sources for %q not found`, importPath) } - return compiler.PrepareSources(srcs, importer, tContext) + return srcs.Prepare(importer, sizes, tContext) } func (s *Session) compilePackages(rootSrcs *sources.Sources, tContext *types.Context) (*compiler.Archive, error) { @@ -1174,6 +1173,20 @@ func (s *Session) getImportPath(path, srcDir string) (string, error) { return pkg.ImportPath, nil } +func (s *Session) SourcesForImport(path, srcDir string) (*sources.Sources, error) { + importPath, err := s.getImportPath(path, srcDir) + if err != nil { + return nil, err + } + + srcs, ok := s.sources[importPath] + if !ok { + return nil, fmt.Errorf(`sources for %q not found`, path) + } + + return srcs, nil +} + // ImportResolverFor returns a function which returns a compiled package archive // given an import path. func (s *Session) ImportResolverFor(srcDir string) func(string) (*compiler.Archive, error) { diff --git a/compiler/compiler_test.go b/compiler/compiler_test.go index ab11c19cc..18ce912bf 100644 --- a/compiler/compiler_test.go +++ b/compiler/compiler_test.go @@ -743,6 +743,10 @@ func reloadCompiledProject(t *testing.T, archives map[string]*Archive, rootPkgPa srcModTime := newTime(0.0) reloadCache := map[string]*Archive{} + type ImportContext struct { + Packages map[string]*types.Package + ImportArchive func(path string) (*Archive, error) + } var importContext *ImportContext importContext = &ImportContext{ Packages: map[string]*types.Package{}, diff --git a/compiler/package.go b/compiler/package.go index 07ec412f4..f87cbad2c 100644 --- a/compiler/package.go +++ b/compiler/package.go @@ -15,7 +15,6 @@ import ( "github.com/gopherjs/gopherjs/compiler/sources" "github.com/gopherjs/gopherjs/compiler/typesutil" "github.com/gopherjs/gopherjs/internal/errorList" - "github.com/gopherjs/gopherjs/internal/experiments" ) // pkgContext maintains compiler context for a specific package. @@ -151,28 +150,33 @@ type flowData struct { endCase int } -// PrepareSources recursively processes the provided sources and -// prepares them for compilation by determining the type information, -// go linknames, and simplifying the AST. -// -// Note that at the end of this call the analysis information -// has NOT been propagated across packages yet. -func PrepareSources(srcs *sources.Sources, importer sources.SourcesImporter, tContext *types.Context) error { - if err := srcs.TypeCheck(importer, sizes32, tContext); err != nil { - return err - } +type SourcesImporter func(path, srcDir string) (*sources.Sources, error) - if genErr := typeparams.RequiresGenericsSupport(srcs.TypeInfo.Info); genErr != nil && !experiments.Env.Generics { - return fmt.Errorf("package %s requires generics support (https://github.com/gopherjs/gopherjs/issues/1013): %w", srcs.ImportPath, genErr) - } +func PrepareAllSources(root *sources.Sources, importer SourcesImporter, tContext *types.Context) error { + +} + +func prepareSources(srcs *sources.Sources, importer SourcesImporter, tContext *types.Context) { + importer := func(path string) (*sources.Sources, error) { + importPath, err := s.getImportPath(path, srcs.Dir) + if err != nil { + return nil, err + } + + if srcs, ok := s.sources[importPath]; ok { + if srcs.Package == nil { + err = s.prepareSources(srcs, tContext) + if err != nil { + return nil, err + } + } + return srcs, nil + } - // Extract all go:linkname compiler directives from the package source. - if err := srcs.ParseGoLinknames(); err != nil { - return err + return nil, fmt.Errorf(`sources for %q not found`, importPath) } - srcs.Simplify() - return nil + return srcs.Prepare(importer, sizes, tContext) } // PropagateAnalysis the analysis information to all packages. diff --git a/compiler/sources/errorCollectingImporter.go b/compiler/sources/errorCollectingImporter.go index a3bdb246c..4b6bc58ff 100644 --- a/compiler/sources/errorCollectingImporter.go +++ b/compiler/sources/errorCollectingImporter.go @@ -9,19 +9,15 @@ import ( // errorCollectingImporter implements go/types.Importer interface and // wraps it to collect import errors. type errorCollectingImporter struct { - Importer SourcesImporter + Importer types.Importer Errors errorList.ErrorList } func (ei *errorCollectingImporter) Import(path string) (*types.Package, error) { - if path == "unsafe" { - return types.Unsafe, nil - } - - srcs, err := ei.Importer(path) + pkg, err := ei.Importer(path) if err != nil { ei.Errors = ei.Errors.AppendDistinct(err) return nil, err } - return srcs.Package, nil + return pkg, nil } diff --git a/compiler/sources/sources.go b/compiler/sources/sources.go index a989bcc59..4bdae3c14 100644 --- a/compiler/sources/sources.go +++ b/compiler/sources/sources.go @@ -1,6 +1,7 @@ package sources import ( + "fmt" "go/ast" "go/token" "go/types" @@ -12,6 +13,7 @@ import ( "github.com/gopherjs/gopherjs/compiler/jsFile" "github.com/gopherjs/gopherjs/compiler/linkname" "github.com/gopherjs/gopherjs/internal/errorList" + "github.com/gopherjs/gopherjs/internal/experiments" "github.com/neelance/astrewrite" ) @@ -42,52 +44,92 @@ type Sources struct { JSFiles []jsFile.JSFile // TypeInfo is the type information this package. - // This is nil until TypeCheck is called. + // This is nil until PrepareInfo is called. TypeInfo *analysis.Info // Instances is the type parameters instances for this package. - // This is nil until TypeCheck is called. + // This is nil until PrepareInfo is called. Instances *typeparams.PackageInstanceSets - // Package is the type-checked package. - // This is nil until TypeCheck is called. + // Package is the type-PrepareInfo package. + // This is nil until PrepareInfo is called. Package *types.Package // GoLinknames is the set of Go linknames for this package. - // This is nil until ParseGoLinknames is called. + // This is nil until PrepareInfo is called. GoLinknames []linkname.GoLinkname } -type SourcesImporter func(path string) (*Sources, error) +type SourcesImporter func(path, srcDir string) (*Sources, error) -// Sort the Go files slice by the original source name to ensure consistent order +// Prepare recursively processes the provided sources and +// prepares them for compilation by sorting the files by name, +// determining the type information, go linknames, and simplifying the AST. +// +// The importer function is used to import the sources of other packages +// that are imported by the package being prepared. The other sources must +// be prepared prior to being returned by the importer so that the type +// information can be used. +// +// Note that at the end of this call the analysis information +// has NOT been propagated across packages yet. +func (s *Sources) Prepare(importer SourcesImporter, sizes types.Sizes, tContext *types.Context) error { + s.sort() + + typesImporter := func(path string) (*types.Package, error) { + if path == "unsafe" { + return types.Unsafe, nil + } + + } + + typesInfo, err := s.typeCheck(typesImporter, sizes, tContext) + if err != nil { + return err + } + + if !experiments.Env.Generics { + if genErr := typeparams.RequiresGenericsSupport(typesInfo); genErr != nil { + return fmt.Errorf("package %s requires generics support (https://github.com/gopherjs/gopherjs/issues/1013): %w", s.ImportPath, genErr) + } + } + + s.analyze(typesInfo, importer, tContext) + + // Extract all go:linkname compiler directives from the package source. + if err := s.parseGoLinknames(); err != nil { + return err + } + + s.simplify() + return nil +} + +// sort the Go files slice by the original source name to ensure consistent order // of processing. This is required for reproducible JavaScript output. // // Note this function mutates the original Files slice. -func (s Sources) Sort() { +func (s Sources) sort() { sort.Slice(s.Files, func(i, j int) bool { return s.FileSet.File(s.Files[i].Pos()).Name() > s.FileSet.File(s.Files[j].Pos()).Name() }) } -// Simplify processed each Files entry with astrewrite.Simplify. +// simplify processed each Files entry with astrewrite.Simplify. // // Note this function mutates the original Files slice. // This must be called after TypeCheck. -func (s Sources) Simplify() { +func (s Sources) simplify() { for i, file := range s.Files { s.Files[i] = astrewrite.Simplify(file, s.TypeInfo.Info, false) } } -// TypeCheck the sources. Returns information about declared package types and +// typeCheck the sources. Returns information about declared package types and // type information for the supplied AST. // -// This will set the Info and Package fields on the Sources struct. // This must be called prior to Simplify. -// Note that at the end of this call the analysis information -// has NOT been propagated across packages yet. -func (s Sources) TypeCheck(importer SourcesImporter, sizes types.Sizes, tContext *types.Context) error { +func (s Sources) typeCheck(importer types.Importer, sizes types.Sizes, tContext *types.Context) (*types.Info, error) { const errLimit = 10 // Max number of type checking errors to return. typesInfo := &types.Info{ @@ -115,48 +157,53 @@ func (s Sources) TypeCheck(importer SourcesImporter, sizes types.Sizes, tContext // are not meaningful and would be resolved by fixing imports. Return them // separately, if any. https://github.com/gopherjs/gopherjs/issues/119. if ecImporter.Errors.ErrOrNil() != nil { - return ecImporter.Errors.Trim(errLimit).ErrOrNil() + return nil, ecImporter.Errors.Trim(errLimit).ErrOrNil() } // Return any other type errors. if typeErrs.ErrOrNil() != nil { - return typeErrs.Trim(errLimit).ErrOrNil() + return nil, typeErrs.Trim(errLimit).ErrOrNil() } // Any general errors that may have occurred during type checking. if err != nil { - return err + return nil, err } + s.Package = typesPkg + return typesInfo, nil +} + +// analyze will determine the type parameters instances, blocking, +// and other type information for the package. +// +// This must be called prior to Simplify. +// Note that at the end of this call the analysis information +// has NOT been propagated across packages yet. +func (s Sources) analyze(typesInfo *types.Info, importer SourcesImporter, tContext *types.Context) { tc := typeparams.Collector{ TContext: tContext, Info: typesInfo, Instances: &typeparams.PackageInstanceSets{}, } - tc.Scan(typesPkg, s.Files...) + tc.Scan(s.Package, s.Files...) - infoImporter := getInfoImporter(importer) - anaInfo := analysis.AnalyzePkg(s.Files, s.FileSet, typesInfo, tContext, typesPkg, tc.Instances, infoImporter) - - s.TypeInfo = anaInfo - s.Instances = tc.Instances - s.Package = typesPkg - return nil -} - -func getInfoImporter(importer SourcesImporter) analysis.InfoImporter { - return func(path string) (*analysis.Info, error) { + infoImporter := func(path string) (*analysis.Info, error) { srcs, err := importer(path) if err != nil { return nil, err } return srcs.TypeInfo, nil } + anaInfo := analysis.AnalyzePkg(s.Files, s.FileSet, typesInfo, tContext, s.Package, tc.Instances, infoImporter) + + s.TypeInfo = anaInfo + s.Instances = tc.Instances } -// ParseGoLinknames extracts all //go:linkname compiler directive from the sources. +// parseGoLinknames extracts all //go:linkname compiler directive from the sources. // // This will set the GoLinknames field on the Sources struct. // This must be called prior to Simplify. -func (s Sources) ParseGoLinknames() error { +func (s Sources) parseGoLinknames() error { goLinknames := []linkname.GoLinkname{} var errs errorList.ErrorList for _, file := range s.Files { From 3def5f3a6a4725846e954197b27a0526296e80bb Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Fri, 28 Feb 2025 16:47:53 -0700 Subject: [PATCH 08/20] Reworking preparing and analyzing sources --- build/build.go | 28 ++------------- compiler/compiler_test.go | 62 ++++++++++++++++----------------- compiler/decls.go | 2 +- compiler/package.go | 33 +++--------------- compiler/sources/sources.go | 69 ++++++++++++++++++------------------- 5 files changed, 70 insertions(+), 124 deletions(-) diff --git a/build/build.go b/build/build.go index 6783c09c5..9ea8d7639 100644 --- a/build/build.go +++ b/build/build.go @@ -932,7 +932,8 @@ func (s *Session) BuildProject(pkg *PackageData) (*compiler.Archive, error) { // Prepare and analyze the source code. // This will be performed recursively for all dependencies. tContext := types.NewContext() - if err = s.prepareSources(srcs, tContext); err != nil { + err = compiler.PrepareAllSources(srcs, s.SourcesForImport, tContext) + if err != nil { return nil, err } @@ -1107,32 +1108,9 @@ func (s *Session) loadPackages(pkg *PackageData) (*sources.Sources, error) { return srcs, nil } -func (s *Session) prepareSources(srcs *sources.Sources, tContext *types.Context) error { - importer := func(path string) (*sources.Sources, error) { - importPath, err := s.getImportPath(path, srcs.Dir) - if err != nil { - return nil, err - } - - if srcs, ok := s.sources[importPath]; ok { - if srcs.Package == nil { - err = s.prepareSources(srcs, tContext) - if err != nil { - return nil, err - } - } - return srcs, nil - } - - return nil, fmt.Errorf(`sources for %q not found`, importPath) - } - - return srcs.Prepare(importer, sizes, tContext) -} - func (s *Session) compilePackages(rootSrcs *sources.Sources, tContext *types.Context) (*compiler.Archive, error) { for _, srcs := range s.sources { - archive, err := compiler.Compile(*srcs, tContext, s.options.Minify) + archive, err := compiler.Compile(srcs, tContext, s.options.Minify) if err != nil { return nil, err } diff --git a/compiler/compiler_test.go b/compiler/compiler_test.go index 18ce912bf..487fe7922 100644 --- a/compiler/compiler_test.go +++ b/compiler/compiler_test.go @@ -679,44 +679,40 @@ func compileProject(t *testing.T, root *packages.Package, minify bool) map[strin pkgMap[pkg.PkgPath] = pkg }) - archiveCache := map[string]*Archive{} - var importContext *ImportContext - importContext = &ImportContext{ - Packages: map[string]*types.Package{}, - ImportArchive: func(path string) (*Archive, error) { - // find in local cache - if a, ok := archiveCache[path]; ok { - return a, nil - } + allSrcs := map[string]*sources.Sources{} + for _, pkg := range pkgMap { + srcs := &sources.Sources{ + ImportPath: pkg.PkgPath, + Dir: ``, + Files: pkg.Syntax, + FileSet: pkg.Fset, + } + allSrcs[pkg.PkgPath] = srcs + } - pkg, ok := pkgMap[path] - if !ok { - t.Fatal(`package not found:`, path) - } - importContext.Packages[path] = pkg.Types + importer := func(path, srcDir string) (*sources.Sources, error) { + srcs, ok := allSrcs[path] + if !ok { + t.Fatal(`package not found:`, path) + return nil, nil + } + return srcs, nil + } - srcs := sources.Sources{ - ImportPath: path, - Files: pkg.Syntax, - FileSet: pkg.Fset, - } - srcs.Sort() + tContext := types.NewContext() + PrepareAllSources(allSrcs[root.PkgPath], importer, tContext) + PropagateAnalysis(allSrcs) - // compile package - a, err := Compile(srcs, importContext, minify) - if err != nil { - return nil, err - } - archiveCache[path] = a - return a, nil - }, + archives := map[string]*Archive{} + for _, srcs := range allSrcs { + a, err := Compile(srcs, tContext, minify) + if err != nil { + t.Fatal(`failed to compile:`, err) + } + archives[srcs.ImportPath] = a } - _, err := importContext.ImportArchive(root.PkgPath) - if err != nil { - t.Fatal(`failed to compile:`, err) - } - return archiveCache + return archives } // newTime creates an arbitrary time.Time offset by the given number of seconds. diff --git a/compiler/decls.go b/compiler/decls.go index 0694181f6..5b760fb15 100644 --- a/compiler/decls.go +++ b/compiler/decls.go @@ -82,7 +82,7 @@ func (d *Decl) Dce() *dce.Info { // topLevelObjects extracts package-level variables, functions and named types // from the package AST. -func (fc *funcContext) topLevelObjects(srcs sources.Sources) (vars []*types.Var, functions []*ast.FuncDecl, typeNames typesutil.TypeNames) { +func (fc *funcContext) topLevelObjects(srcs *sources.Sources) (vars []*types.Var, functions []*ast.FuncDecl, typeNames typesutil.TypeNames) { if !fc.isRoot() { panic(bailout(fmt.Errorf("functionContext.discoverObjects() must be only called on the package-level context"))) } diff --git a/compiler/package.go b/compiler/package.go index f87cbad2c..f21d6cce3 100644 --- a/compiler/package.go +++ b/compiler/package.go @@ -116,7 +116,7 @@ type funcContext struct { funcLitCounter int } -func newRootCtx(tContext *types.Context, srcs sources.Sources, minify bool) *funcContext { +func newRootCtx(tContext *types.Context, srcs *sources.Sources, minify bool) *funcContext { funcCtx := &funcContext{ FuncInfo: srcs.TypeInfo.InitFuncInfo, pkgCtx: &pkgContext{ @@ -150,33 +150,8 @@ type flowData struct { endCase int } -type SourcesImporter func(path, srcDir string) (*sources.Sources, error) - -func PrepareAllSources(root *sources.Sources, importer SourcesImporter, tContext *types.Context) error { - -} - -func prepareSources(srcs *sources.Sources, importer SourcesImporter, tContext *types.Context) { - importer := func(path string) (*sources.Sources, error) { - importPath, err := s.getImportPath(path, srcs.Dir) - if err != nil { - return nil, err - } - - if srcs, ok := s.sources[importPath]; ok { - if srcs.Package == nil { - err = s.prepareSources(srcs, tContext) - if err != nil { - return nil, err - } - } - return srcs, nil - } - - return nil, fmt.Errorf(`sources for %q not found`, importPath) - } - - return srcs.Prepare(importer, sizes, tContext) +func PrepareAllSources(root *sources.Sources, importer sources.Importer, tContext *types.Context) error { + return root.Prepare(importer, sizes32, tContext) } // PropagateAnalysis the analysis information to all packages. @@ -194,7 +169,7 @@ func PropagateAnalysis(allSources map[string]*sources.Sources) { // Compile the provided Go sources as a single package. // // Provided sources must be sorted by name to ensure reproducible JavaScript output. -func Compile(srcs sources.Sources, tContext *types.Context, minify bool) (_ *Archive, err error) { +func Compile(srcs *sources.Sources, tContext *types.Context, minify bool) (_ *Archive, err error) { defer func() { e := recover() if e == nil { diff --git a/compiler/sources/sources.go b/compiler/sources/sources.go index a2886dbb4..017cba0e6 100644 --- a/compiler/sources/sources.go +++ b/compiler/sources/sources.go @@ -60,7 +60,7 @@ type Sources struct { GoLinknames []linkname.GoLinkname } -type SourcesImporter func(path, srcDir string) (*Sources, error) +type Importer func(path, srcDir string) (*Sources, error) // Prepare recursively processes the provided sources and // prepares them for compilation by sorting the files by name, @@ -73,17 +73,10 @@ type SourcesImporter func(path, srcDir string) (*Sources, error) // // Note that at the end of this call the analysis information // has NOT been propagated across packages yet. -func (s *Sources) Prepare(importer SourcesImporter, sizes types.Sizes, tContext *types.Context) error { +func (s *Sources) Prepare(importer Importer, sizes types.Sizes, tContext *types.Context) error { s.sort() - typesImporter := func(path string) (*types.Package, error) { - if path == "unsafe" { - return types.Unsafe, nil - } - - } - - typesInfo, err := s.typeCheck(typesImporter, sizes, tContext) + typesInfo, err := s.typeCheck(importer, sizes, tContext) if err != nil { return err } @@ -109,7 +102,7 @@ func (s *Sources) Prepare(importer SourcesImporter, sizes types.Sizes, tContext // of processing. This is required for reproducible JavaScript output. // // Note this function mutates the original Files slice. -func (s Sources) sort() { +func (s *Sources) sort() { sort.Slice(s.Files, func(i, j int) bool { return s.FileSet.File(s.Files[i].Pos()).Name() > s.FileSet.File(s.Files[j].Pos()).Name() }) @@ -119,7 +112,7 @@ func (s Sources) sort() { // // Note this function mutates the original Files slice. // This must be called after TypeCheck. -func (s Sources) simplify() { +func (s *Sources) simplify() { for i, file := range s.Files { s.Files[i] = astrewrite.Simplify(file, s.TypeInfo.Info, false) } @@ -129,7 +122,7 @@ func (s Sources) simplify() { // type information for the supplied AST. // // This must be called prior to Simplify. -func (s Sources) typeCheck(importer types.Importer, sizes types.Sizes, tContext *types.Context) (*types.Info, error) { +func (s *Sources) typeCheck(importer Importer, sizes types.Sizes, tContext *types.Context) (*types.Info, error) { const errLimit = 10 // Max number of type checking errors to return. typesInfo := &types.Info{ @@ -144,11 +137,16 @@ func (s Sources) typeCheck(importer types.Importer, sizes types.Sizes, tContext var typeErrs errorList.ErrorList - ecImporter := &packageImporter{Importer: importer} + pkgImporter := &packageImporter{ + srcDir: s.Dir, + importer: importer, + sizes: sizes, + tContext: tContext, + } config := &types.Config{ Context: tContext, - Importer: ecImporter, + Importer: pkgImporter, Sizes: sizes, Error: func(err error) { typeErrs = typeErrs.AppendDistinct(err) }, } @@ -156,8 +154,8 @@ func (s Sources) typeCheck(importer types.Importer, sizes types.Sizes, tContext // If we encountered any import errors, it is likely that the other type errors // are not meaningful and would be resolved by fixing imports. Return them // separately, if any. https://github.com/gopherjs/gopherjs/issues/119. - if ecImporter.Errors.ErrOrNil() != nil { - return nil, ecImporter.Errors.Trim(errLimit).ErrOrNil() + if pkgImporter.Errors.ErrOrNil() != nil { + return nil, pkgImporter.Errors.Trim(errLimit).ErrOrNil() } // Return any other type errors. if typeErrs.ErrOrNil() != nil { @@ -178,7 +176,7 @@ func (s Sources) typeCheck(importer types.Importer, sizes types.Sizes, tContext // This must be called prior to Simplify. // Note that at the end of this call the analysis information // has NOT been propagated across packages yet. -func (s Sources) analyze(typesInfo *types.Info, importer SourcesImporter, tContext *types.Context) { +func (s *Sources) analyze(typesInfo *types.Info, importer Importer, tContext *types.Context) { tc := typeparams.Collector{ TContext: tContext, Info: typesInfo, @@ -187,7 +185,7 @@ func (s Sources) analyze(typesInfo *types.Info, importer SourcesImporter, tConte tc.Scan(s.Package, s.Files...) infoImporter := func(path string) (*analysis.Info, error) { - srcs, err := importer(path) + srcs, err := importer(path, s.Dir) if err != nil { return nil, err } @@ -203,7 +201,7 @@ func (s Sources) analyze(typesInfo *types.Info, importer SourcesImporter, tConte // // This will set the GoLinknames field on the Sources struct. // This must be called prior to Simplify. -func (s Sources) parseGoLinknames() error { +func (s *Sources) parseGoLinknames() error { goLinknames := []linkname.GoLinkname{} var errs errorList.ErrorList for _, file := range s.Files { @@ -218,18 +216,6 @@ func (s Sources) parseGoLinknames() error { return nil } -// ParseGoLinknames extracts all //go:linkname compiler directive from the sources. -func (s Sources) ParseGoLinknames() ([]linkname.GoLinkname, error) { - goLinknames := []linkname.GoLinkname{} - var errs errorList.ErrorList - for _, file := range s.Files { - found, err := linkname.ParseGoLinknames(s.FileSet, s.ImportPath, file) - errs = errs.Append(err) - goLinknames = append(goLinknames, found...) - } - return goLinknames, errs.ErrOrNil() -} - // UnresolvedImports calculates the import paths of the package's dependencies // based on all the imports in the augmented Go AST files. // @@ -240,7 +226,7 @@ func (s Sources) ParseGoLinknames() ([]linkname.GoLinkname, error) { // The given skip paths (typically those imports from PackageData.Imports) // will not be returned in the results. // This will not return any `*_test` packages in the results. -func (s Sources) UnresolvedImports(skip ...string) []string { +func (s *Sources) UnresolvedImports(skip ...string) []string { seen := make(map[string]struct{}) for _, sk := range skip { seen[sk] = struct{}{} @@ -264,7 +250,10 @@ func (s Sources) UnresolvedImports(skip ...string) []string { // packageImporter implements go/types.Importer interface and // wraps it to collect import errors. type packageImporter struct { - Importer types.Importer + srcDir string + importer Importer + sizes types.Sizes + tContext *types.Context Errors errorList.ErrorList } @@ -273,11 +262,19 @@ func (ei *packageImporter) Import(path string) (*types.Package, error) { return types.Unsafe, nil } - pkg, err := ei.Importer.Import(path) + srcs, err := ei.importer(path, ei.srcDir) if err != nil { ei.Errors = ei.Errors.AppendDistinct(err) return nil, err } - return pkg, nil + if srcs.Package == nil { + err := srcs.Prepare(ei.importer, ei.sizes, ei.tContext) + if err != nil { + ei.Errors = ei.Errors.AppendDistinct(err) + return nil, err + } + } + + return srcs.Package, nil } From 731f220394f1ffc617fcb01bb002f9d3361d5483 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Mon, 3 Mar 2025 14:56:40 -0700 Subject: [PATCH 09/20] Trying to debug the funcLit pointer issue --- build/build.go | 5 +--- compiler/compiler_test.go | 3 +-- compiler/internal/analysis/info.go | 24 ++++++++++++++++++- compiler/internal/analysis/info_test.go | 31 ++++++++++++++----------- compiler/package.go | 20 +++++++++------- compiler/sources/sources.go | 19 ++++++++------- 6 files changed, 64 insertions(+), 38 deletions(-) diff --git a/build/build.go b/build/build.go index 9ea8d7639..5b1ff9ea3 100644 --- a/build/build.go +++ b/build/build.go @@ -932,14 +932,11 @@ func (s *Session) BuildProject(pkg *PackageData) (*compiler.Archive, error) { // Prepare and analyze the source code. // This will be performed recursively for all dependencies. tContext := types.NewContext() - err = compiler.PrepareAllSources(srcs, s.SourcesForImport, tContext) + err = compiler.PrepareAllSources(srcs, s.sources, s.SourcesForImport, tContext) if err != nil { return nil, err } - // Propagate the analysis information to all packages. - compiler.PropagateAnalysis(s.sources) - // Compile the project into Archives containing the generated JS. return s.compilePackages(srcs, tContext) } diff --git a/compiler/compiler_test.go b/compiler/compiler_test.go index 487fe7922..ca0d2c69a 100644 --- a/compiler/compiler_test.go +++ b/compiler/compiler_test.go @@ -700,8 +700,7 @@ func compileProject(t *testing.T, root *packages.Package, minify bool) map[strin } tContext := types.NewContext() - PrepareAllSources(allSrcs[root.PkgPath], importer, tContext) - PropagateAnalysis(allSrcs) + PrepareAllSources(allSrcs[root.PkgPath], allSrcs, importer, tContext) archives := map[string]*Archive{} for _, srcs := range allSrcs { diff --git a/compiler/internal/analysis/info.go b/compiler/internal/analysis/info.go index f5bebc91d..f218f9441 100644 --- a/compiler/internal/analysis/info.go +++ b/compiler/internal/analysis/info.go @@ -1,8 +1,10 @@ package analysis import ( + "bytes" "fmt" "go/ast" + "go/printer" "go/token" "go/types" "strings" @@ -98,6 +100,13 @@ func (info *Info) newFuncInfo(n ast.Node, obj types.Object, typeArgs typesutil.T case *ast.FuncLit: info.funcLitInfos[n] = append(info.funcLitInfos[n], funcInfo) + + // TODO(grantnelson-wf): Remove below + buf := &bytes.Buffer{} + fSet := token.NewFileSet() + printer.Fprint(buf, fSet, n) + fmt.Printf(">>|<< Adding %q with [%v] in package %s\n", buf.String(), typeArgs, info.Pkg.Path()) + fmt.Printf(">>|<< \tHad: %v\n", info.funcLitInfos[n]) } // And add it to the list of all functions. @@ -166,7 +175,15 @@ func (info *Info) FuncLitInfo(fun *ast.FuncLit, typeArgs typesutil.TypeList) *Fu return fi } } - return nil + + // TODO(grantnelson-wf): Remove below + buf := &bytes.Buffer{} + fSet := token.NewFileSet() + printer.Fprint(buf, fSet, fun) + fmt.Printf(">>|<< Failed to find %q (%p) with [%v] in package %s\n", buf.String(), fun, typeArgs, info.Pkg.Path()) + fmt.Printf(">>|<< \tHad: %v\n\n", lits) + panic("BOOM!") + //return nil } // VarsWithInitializers returns a set of package-level variables that have @@ -256,6 +273,11 @@ func (info *Info) propagateFunctionBlocking() bool { } } } + + // TODO(grantnelson-wf): Remove below + fmt.Printf(">>|<< propagateFunctionBlocking package %s\n", info.Pkg.Path()) + fmt.Printf(">>|<< \tHad: %v\n", info.funcLitInfos) + return done } diff --git a/compiler/internal/analysis/info_test.go b/compiler/internal/analysis/info_test.go index 0e83a04bc..e6eaf0dec 100644 --- a/compiler/internal/analysis/info_test.go +++ b/compiler/internal/analysis/info_test.go @@ -1576,15 +1576,13 @@ func TestBlocking_IsImportBlocking_ForwardInstances(t *testing.T) { } func TestBlocking_IsImportBlocking_BackwardInstances(t *testing.T) { - t.Skip(`isImportedBlocking doesn't fully handle instances yet`) - // TODO(grantnelson-wf): This test is currently failing because the info - // for the test package is need while creating the instances for FooBaz - // while analyzing the other package. However the other package is analyzed - // first since the test package is dependent on it. One possible fix is that - // we add some mechanism similar to the localInstCallees but for remote - // instances then perform the blocking propagation steps for all packages - // including the localInstCallees propagation at the same time. After all the - // propagation of the calls then the flow control statements can be marked. + // This is tests propagation of information across package boundaries. + // `FooBaz` has no instances in it until it is referenced in the `test` package. + // That instance information needs to propagate back across the package + // boundary to the `other` package. The information for `BazBlocker` and + // `BazNotBlocker` is propagated back to `FooBaz[BazBlocker]` and + // `FooBaz[BazNotBlocker]`. That information is then propagated forward + // to the `blocking` and `notBlocking` functions in the `test` package. otherSrc := `package other @@ -1630,8 +1628,9 @@ type blockingTest struct { func newBlockingTest(t *testing.T, src string) *blockingTest { f := srctesting.New(t) + tContext := types.NewContext() tc := typeparams.Collector{ - TContext: types.NewContext(), + TContext: tContext, Info: f.Info, Instances: &typeparams.PackageInstanceSets{}, } @@ -1643,7 +1642,8 @@ func newBlockingTest(t *testing.T, src string) *blockingTest { getImportInfo := func(path string) (*Info, error) { return nil, fmt.Errorf(`getImportInfo should not be called in this test, called with %v`, path) } - pkgInfo := AnalyzePkg([]*ast.File{file}, f.FileSet, testInfo, types.NewContext(), testPkg, tc.Instances, getImportInfo) + pkgInfo := AnalyzePkg([]*ast.File{file}, f.FileSet, testInfo, tContext, testPkg, tc.Instances, getImportInfo) + PropagateAnalysis([]*Info{pkgInfo}) return &blockingTest{ f: f, @@ -1654,8 +1654,9 @@ func newBlockingTest(t *testing.T, src string) *blockingTest { func newBlockingTestWithOtherPackage(t *testing.T, testSrc string, otherSrc string) *blockingTest { f := srctesting.New(t) + tContext := types.NewContext() tc := typeparams.Collector{ - TContext: types.NewContext(), + TContext: tContext, Info: f.Info, Instances: &typeparams.PackageInstanceSets{}, } @@ -1676,12 +1677,14 @@ func newBlockingTestWithOtherPackage(t *testing.T, testSrc string, otherSrc stri _, testPkg := f.Check(`pkg/test`, testFile) tc.Scan(testPkg, testFile) - otherPkgInfo := AnalyzePkg([]*ast.File{otherFile}, f.FileSet, f.Info, types.NewContext(), otherPkg, tc.Instances, getImportInfo) + otherPkgInfo := AnalyzePkg([]*ast.File{otherFile}, f.FileSet, f.Info, tContext, otherPkg, tc.Instances, getImportInfo) pkgInfo[otherPkg.Path()] = otherPkgInfo - testPkgInfo := AnalyzePkg([]*ast.File{testFile}, f.FileSet, f.Info, types.NewContext(), testPkg, tc.Instances, getImportInfo) + testPkgInfo := AnalyzePkg([]*ast.File{testFile}, f.FileSet, f.Info, tContext, testPkg, tc.Instances, getImportInfo) pkgInfo[testPkg.Path()] = testPkgInfo + PropagateAnalysis([]*Info{otherPkgInfo, testPkgInfo}) + return &blockingTest{ f: f, file: testFile, diff --git a/compiler/package.go b/compiler/package.go index f21d6cce3..fe68120e2 100644 --- a/compiler/package.go +++ b/compiler/package.go @@ -150,20 +150,24 @@ type flowData struct { endCase int } -func PrepareAllSources(root *sources.Sources, importer sources.Importer, tContext *types.Context) error { - return root.Prepare(importer, sizes32, tContext) -} +func PrepareAllSources(root *sources.Sources, allSources map[string]*sources.Sources, importer sources.Importer, tContext *types.Context) error { + // This will be performed recursively for all dependencies starting from the root. + if err := root.Prepare(importer, sizes32, tContext); err != nil { + return err + } -// PropagateAnalysis the analysis information to all packages. -// -// The map of sources is keyed by the resolved import path of the package, -// however that key is not used. -func PropagateAnalysis(allSources map[string]*sources.Sources) { + // Propagate the analysis information to all packages. allInfo := make([]*analysis.Info, 0, len(allSources)) for _, src := range allSources { allInfo = append(allInfo, src.TypeInfo) } analysis.PropagateAnalysis(allInfo) + + // Simplify the source files. + for _, srcs := range allSources { + srcs.Simplify() + } + return nil } // Compile the provided Go sources as a single package. diff --git a/compiler/sources/sources.go b/compiler/sources/sources.go index 017cba0e6..b9c6341e8 100644 --- a/compiler/sources/sources.go +++ b/compiler/sources/sources.go @@ -64,7 +64,7 @@ type Importer func(path, srcDir string) (*Sources, error) // Prepare recursively processes the provided sources and // prepares them for compilation by sorting the files by name, -// determining the type information, go linknames, and simplifying the AST. +// determining the type information, go linknames, etc. // // The importer function is used to import the sources of other packages // that are imported by the package being prepared. The other sources must @@ -72,30 +72,31 @@ type Importer func(path, srcDir string) (*Sources, error) // information can be used. // // Note that at the end of this call the analysis information -// has NOT been propagated across packages yet. +// has NOT been propagated across packages yet +// and the source files have not been simplified yet. func (s *Sources) Prepare(importer Importer, sizes types.Sizes, tContext *types.Context) error { + // Sort the files by name to ensure consistent order of processing. s.sort() + // Type check the sources to determine the type information. typesInfo, err := s.typeCheck(importer, sizes, tContext) if err != nil { return err } + // If generics are not enabled, ensure the package does not requires generics support. if !experiments.Env.Generics { if genErr := typeparams.RequiresGenericsSupport(typesInfo); genErr != nil { return fmt.Errorf("package %s requires generics support (https://github.com/gopherjs/gopherjs/issues/1013): %w", s.ImportPath, genErr) } } + // Analyze the package to determine type parameters instances, blocking, + // and other type information. This will not populate the information. s.analyze(typesInfo, importer, tContext) // Extract all go:linkname compiler directives from the package source. - if err := s.parseGoLinknames(); err != nil { - return err - } - - s.simplify() - return nil + return s.parseGoLinknames() } // sort the Go files slice by the original source name to ensure consistent order @@ -112,7 +113,7 @@ func (s *Sources) sort() { // // Note this function mutates the original Files slice. // This must be called after TypeCheck. -func (s *Sources) simplify() { +func (s *Sources) Simplify() { for i, file := range s.Files { s.Files[i] = astrewrite.Simplify(file, s.TypeInfo.Info, false) } From ae7afcd044690aab24f93ef233d30fee6334c77a Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 4 Mar 2025 10:09:16 -0700 Subject: [PATCH 10/20] Fixing the funclit pointer problem --- compiler/internal/analysis/info.go | 24 +----------------------- compiler/package.go | 5 ----- compiler/sources/sources.go | 27 +++++++++++++++++++-------- 3 files changed, 20 insertions(+), 36 deletions(-) diff --git a/compiler/internal/analysis/info.go b/compiler/internal/analysis/info.go index f218f9441..f5bebc91d 100644 --- a/compiler/internal/analysis/info.go +++ b/compiler/internal/analysis/info.go @@ -1,10 +1,8 @@ package analysis import ( - "bytes" "fmt" "go/ast" - "go/printer" "go/token" "go/types" "strings" @@ -100,13 +98,6 @@ func (info *Info) newFuncInfo(n ast.Node, obj types.Object, typeArgs typesutil.T case *ast.FuncLit: info.funcLitInfos[n] = append(info.funcLitInfos[n], funcInfo) - - // TODO(grantnelson-wf): Remove below - buf := &bytes.Buffer{} - fSet := token.NewFileSet() - printer.Fprint(buf, fSet, n) - fmt.Printf(">>|<< Adding %q with [%v] in package %s\n", buf.String(), typeArgs, info.Pkg.Path()) - fmt.Printf(">>|<< \tHad: %v\n", info.funcLitInfos[n]) } // And add it to the list of all functions. @@ -175,15 +166,7 @@ func (info *Info) FuncLitInfo(fun *ast.FuncLit, typeArgs typesutil.TypeList) *Fu return fi } } - - // TODO(grantnelson-wf): Remove below - buf := &bytes.Buffer{} - fSet := token.NewFileSet() - printer.Fprint(buf, fSet, fun) - fmt.Printf(">>|<< Failed to find %q (%p) with [%v] in package %s\n", buf.String(), fun, typeArgs, info.Pkg.Path()) - fmt.Printf(">>|<< \tHad: %v\n\n", lits) - panic("BOOM!") - //return nil + return nil } // VarsWithInitializers returns a set of package-level variables that have @@ -273,11 +256,6 @@ func (info *Info) propagateFunctionBlocking() bool { } } } - - // TODO(grantnelson-wf): Remove below - fmt.Printf(">>|<< propagateFunctionBlocking package %s\n", info.Pkg.Path()) - fmt.Printf(">>|<< \tHad: %v\n", info.funcLitInfos) - return done } diff --git a/compiler/package.go b/compiler/package.go index fe68120e2..1a2ed5c9a 100644 --- a/compiler/package.go +++ b/compiler/package.go @@ -162,11 +162,6 @@ func PrepareAllSources(root *sources.Sources, allSources map[string]*sources.Sou allInfo = append(allInfo, src.TypeInfo) } analysis.PropagateAnalysis(allInfo) - - // Simplify the source files. - for _, srcs := range allSources { - srcs.Simplify() - } return nil } diff --git a/compiler/sources/sources.go b/compiler/sources/sources.go index b9c6341e8..9a531a933 100644 --- a/compiler/sources/sources.go +++ b/compiler/sources/sources.go @@ -91,12 +91,20 @@ func (s *Sources) Prepare(importer Importer, sizes types.Sizes, tContext *types. } } + // Extract all go:linkname compiler directives from the package source. + err = s.parseGoLinknames() + if err != nil { + return err + } + + // Simply the source files. + s.simplify(typesInfo) + // Analyze the package to determine type parameters instances, blocking, // and other type information. This will not populate the information. s.analyze(typesInfo, importer, tContext) - // Extract all go:linkname compiler directives from the package source. - return s.parseGoLinknames() + return nil } // sort the Go files slice by the original source name to ensure consistent order @@ -112,17 +120,20 @@ func (s *Sources) sort() { // simplify processed each Files entry with astrewrite.Simplify. // // Note this function mutates the original Files slice. -// This must be called after TypeCheck. -func (s *Sources) Simplify() { +// This must be called after TypeCheck and before analyze since +// this will change the pointers in the AST, for example the pointers +// to function literals will change making it impossible to find them +// in the type information if analyze is called first. +func (s *Sources) simplify(typesInfo *types.Info) { for i, file := range s.Files { - s.Files[i] = astrewrite.Simplify(file, s.TypeInfo.Info, false) + s.Files[i] = astrewrite.Simplify(file, typesInfo, false) } } // typeCheck the sources. Returns information about declared package types and // type information for the supplied AST. // -// This must be called prior to Simplify. +// This must be called prior to simplify. func (s *Sources) typeCheck(importer Importer, sizes types.Sizes, tContext *types.Context) (*types.Info, error) { const errLimit = 10 // Max number of type checking errors to return. @@ -174,7 +185,7 @@ func (s *Sources) typeCheck(importer Importer, sizes types.Sizes, tContext *type // analyze will determine the type parameters instances, blocking, // and other type information for the package. // -// This must be called prior to Simplify. +// This must be called after to simplify. // Note that at the end of this call the analysis information // has NOT been propagated across packages yet. func (s *Sources) analyze(typesInfo *types.Info, importer Importer, tContext *types.Context) { @@ -201,7 +212,7 @@ func (s *Sources) analyze(typesInfo *types.Info, importer Importer, tContext *ty // parseGoLinknames extracts all //go:linkname compiler directive from the sources. // // This will set the GoLinknames field on the Sources struct. -// This must be called prior to Simplify. +// This must be called prior to simplify. func (s *Sources) parseGoLinknames() error { goLinknames := []linkname.GoLinkname{} var errs errorList.ErrorList From ccb500b62cfa9c5a19fbb49608b3102aa6b00653 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 4 Mar 2025 10:21:13 -0700 Subject: [PATCH 11/20] Fixing the funclit pointer problem --- compiler/package.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/compiler/package.go b/compiler/package.go index 1a2ed5c9a..41fb65025 100644 --- a/compiler/package.go +++ b/compiler/package.go @@ -156,9 +156,22 @@ func PrepareAllSources(root *sources.Sources, allSources map[string]*sources.Sou return err } + for _, srcs := range allSources { + if srcs.TypeInfo == nil { + fmt.Printf("PrepareAllSources: typeInfo is nil for %s\n", srcs.ImportPath) + if err := srcs.Prepare(importer, sizes32, tContext); err != nil { + return err + } + } + } + // Propagate the analysis information to all packages. allInfo := make([]*analysis.Info, 0, len(allSources)) for _, src := range allSources { + typeInfo := src.TypeInfo + if typeInfo == nil { + panic(fmt.Errorf("PrepareAllSources: typeInfo is nil for %s", src.ImportPath)) + } allInfo = append(allInfo, src.TypeInfo) } analysis.PropagateAnalysis(allInfo) From a79a57408e21dedcd73d5482f7c6634ffe708f74 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 4 Mar 2025 10:28:47 -0700 Subject: [PATCH 12/20] Fixing the funclit pointer problem --- compiler/package.go | 22 +++++++--------------- compiler/sources/sources.go | 9 +++++++++ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/compiler/package.go b/compiler/package.go index 41fb65025..c2c0f733c 100644 --- a/compiler/package.go +++ b/compiler/package.go @@ -150,28 +150,20 @@ type flowData struct { endCase int } -func PrepareAllSources(root *sources.Sources, allSources map[string]*sources.Sources, importer sources.Importer, tContext *types.Context) error { - // This will be performed recursively for all dependencies starting from the root. - if err := root.Prepare(importer, sizes32, tContext); err != nil { - return err - } - +func PrepareAllSources(allSources map[string]*sources.Sources, importer sources.Importer, tContext *types.Context) error { + // This will be performed recursively for all dependencies so + // most of these prepare calls will be no-ops. Since not all packages will + // be recursively reached via the root source, we need to prepare them + // all here. for _, srcs := range allSources { - if srcs.TypeInfo == nil { - fmt.Printf("PrepareAllSources: typeInfo is nil for %s\n", srcs.ImportPath) - if err := srcs.Prepare(importer, sizes32, tContext); err != nil { - return err - } + if err := srcs.Prepare(importer, sizes32, tContext); err != nil { + return err } } // Propagate the analysis information to all packages. allInfo := make([]*analysis.Info, 0, len(allSources)) for _, src := range allSources { - typeInfo := src.TypeInfo - if typeInfo == nil { - panic(fmt.Errorf("PrepareAllSources: typeInfo is nil for %s", src.ImportPath)) - } allInfo = append(allInfo, src.TypeInfo) } analysis.PropagateAnalysis(allInfo) diff --git a/compiler/sources/sources.go b/compiler/sources/sources.go index 9a531a933..d6e1fe93b 100644 --- a/compiler/sources/sources.go +++ b/compiler/sources/sources.go @@ -75,6 +75,11 @@ type Importer func(path, srcDir string) (*Sources, error) // has NOT been propagated across packages yet // and the source files have not been simplified yet. func (s *Sources) Prepare(importer Importer, sizes types.Sizes, tContext *types.Context) error { + // Skip if the sources have already been prepared. + if s.isPrepared() { + return nil + } + // Sort the files by name to ensure consistent order of processing. s.sort() @@ -107,6 +112,10 @@ func (s *Sources) Prepare(importer Importer, sizes types.Sizes, tContext *types. return nil } +func (s *Sources) isPrepared() bool { + return s.TypeInfo != nil && s.Package != nil +} + // sort the Go files slice by the original source name to ensure consistent order // of processing. This is required for reproducible JavaScript output. // From 471fbf5d2b46ab3fc2c938511c9af457f1133867 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 4 Mar 2025 10:32:53 -0700 Subject: [PATCH 13/20] Fixing the funclit pointer problem --- build/build.go | 16 +++++++++++++++- compiler/package.go | 8 ++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/build/build.go b/build/build.go index 5b1ff9ea3..844cb4e9e 100644 --- a/build/build.go +++ b/build/build.go @@ -932,7 +932,8 @@ func (s *Session) BuildProject(pkg *PackageData) (*compiler.Archive, error) { // Prepare and analyze the source code. // This will be performed recursively for all dependencies. tContext := types.NewContext() - err = compiler.PrepareAllSources(srcs, s.sources, s.SourcesForImport, tContext) + allSources := s.getSortedSources() + err = compiler.PrepareAllSources(s.sources, s.SourcesForImport, tContext) if err != nil { return nil, err } @@ -941,6 +942,19 @@ func (s *Session) BuildProject(pkg *PackageData) (*compiler.Archive, error) { return s.compilePackages(srcs, tContext) } +// getSortedSources returns the sources sorted by import path. +// The files in the sources may still not be sorted yet. +func (s *Session) getSortedSources() []*sources.Sources { + srcs := make([]*sources.Sources, 0, len(s.sources)) + for _, src := range s.sources { + srcs = append(srcs, src) + } + sort.Slice(srcs, func(i, j int) bool { + return srcs[i].ImportPath < srcs[j].ImportPath + }) + return srcs +} + func (s *Session) loadTestPackage(pkg *PackageData) (*sources.Sources, error) { _, err := s.loadPackages(pkg.TestPackage()) if err != nil { diff --git a/compiler/package.go b/compiler/package.go index c2c0f733c..ec1eb8634 100644 --- a/compiler/package.go +++ b/compiler/package.go @@ -150,7 +150,7 @@ type flowData struct { endCase int } -func PrepareAllSources(allSources map[string]*sources.Sources, importer sources.Importer, tContext *types.Context) error { +func PrepareAllSources(allSources []*sources.Sources, importer sources.Importer, tContext *types.Context) error { // This will be performed recursively for all dependencies so // most of these prepare calls will be no-ops. Since not all packages will // be recursively reached via the root source, we need to prepare them @@ -162,9 +162,9 @@ func PrepareAllSources(allSources map[string]*sources.Sources, importer sources. } // Propagate the analysis information to all packages. - allInfo := make([]*analysis.Info, 0, len(allSources)) - for _, src := range allSources { - allInfo = append(allInfo, src.TypeInfo) + allInfo := make([]*analysis.Info, len(allSources)) + for i, src := range allSources { + allInfo[i] = src.TypeInfo } analysis.PropagateAnalysis(allInfo) return nil From b5ab57ed6a2a2cea34bfd500a6405266bb064eb3 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 4 Mar 2025 10:34:24 -0700 Subject: [PATCH 14/20] Fixing the funclit pointer problem --- build/build.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/build.go b/build/build.go index 844cb4e9e..efae6c26c 100644 --- a/build/build.go +++ b/build/build.go @@ -933,7 +933,7 @@ func (s *Session) BuildProject(pkg *PackageData) (*compiler.Archive, error) { // This will be performed recursively for all dependencies. tContext := types.NewContext() allSources := s.getSortedSources() - err = compiler.PrepareAllSources(s.sources, s.SourcesForImport, tContext) + err = compiler.PrepareAllSources(allSources, s.SourcesForImport, tContext) if err != nil { return nil, err } From 04ab0c0cbb6df853821e0ca22c6d90fb76344048 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 4 Mar 2025 10:41:09 -0700 Subject: [PATCH 15/20] Fixing the funclit pointer problem --- build/build.go | 12 ++++++------ compiler/compiler_test.go | 9 ++++++++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/build/build.go b/build/build.go index efae6c26c..a3e75740b 100644 --- a/build/build.go +++ b/build/build.go @@ -945,14 +945,14 @@ func (s *Session) BuildProject(pkg *PackageData) (*compiler.Archive, error) { // getSortedSources returns the sources sorted by import path. // The files in the sources may still not be sorted yet. func (s *Session) getSortedSources() []*sources.Sources { - srcs := make([]*sources.Sources, 0, len(s.sources)) - for _, src := range s.sources { - srcs = append(srcs, src) + allSources := make([]*sources.Sources, 0, len(s.sources)) + for _, srcs := range s.sources { + allSources = append(allSources, srcs) } - sort.Slice(srcs, func(i, j int) bool { - return srcs[i].ImportPath < srcs[j].ImportPath + sort.Slice(allSources, func(i, j int) bool { + return allSources[i].ImportPath < allSources[j].ImportPath }) - return srcs + return allSources } func (s *Session) loadTestPackage(pkg *PackageData) (*sources.Sources, error) { diff --git a/compiler/compiler_test.go b/compiler/compiler_test.go index ca0d2c69a..d0509b4d4 100644 --- a/compiler/compiler_test.go +++ b/compiler/compiler_test.go @@ -700,7 +700,14 @@ func compileProject(t *testing.T, root *packages.Package, minify bool) map[strin } tContext := types.NewContext() - PrepareAllSources(allSrcs[root.PkgPath], allSrcs, importer, tContext) + sortedSources := make([]*sources.Sources, 0, len(allSrcs)) + for _, srcs := range allSrcs { + sortedSources = append(sortedSources, srcs) + } + sort.Slice(sortedSources, func(i, j int) bool { + return sortedSources[i].ImportPath < sortedSources[j].ImportPath + }) + PrepareAllSources(sortedSources, importer, tContext) archives := map[string]*Archive{} for _, srcs := range allSrcs { From 35e4013c457791260ef554cf4095150826d6b661 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 4 Mar 2025 13:47:50 -0700 Subject: [PATCH 16/20] Fixing remaining bugs --- build/build.go | 45 ++++++++++++++++++++++++++----------- compiler/sources/sources.go | 19 ++++++++-------- 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/build/build.go b/build/build.go index a3e75740b..3deb667e4 100644 --- a/build/build.go +++ b/build/build.go @@ -1121,25 +1121,44 @@ func (s *Session) loadPackages(pkg *PackageData) (*sources.Sources, error) { func (s *Session) compilePackages(rootSrcs *sources.Sources, tContext *types.Context) (*compiler.Archive, error) { for _, srcs := range s.sources { - archive, err := compiler.Compile(srcs, tContext, s.options.Minify) - if err != nil { - return nil, err - } + s.compilePackage(srcs, tContext) + } - for _, jsFile := range srcs.JSFiles { - archive.IncJSCode = append(archive.IncJSCode, []byte("\t(function() {\n")...) - archive.IncJSCode = append(archive.IncJSCode, jsFile.Content...) - archive.IncJSCode = append(archive.IncJSCode, []byte("\n\t}).call($global);\n")...) + rootArchive, ok := s.UpToDateArchives[rootSrcs.ImportPath] + if !ok { + // TODO(grantnelson-wf): Remove all but return + fmt.Printf(">> root package %q not found\n", rootSrcs.ImportPath) + fmt.Printf(">> sources contained:\n") + for _, srcs := range s.sources { + fmt.Printf(">> source: %s\n", srcs.ImportPath) } - if s.options.Verbose { - fmt.Println(srcs.ImportPath) - } + return nil, fmt.Errorf(`root package %q not found`, rootSrcs.ImportPath) + } + return rootArchive, nil +} + +func (s *Session) compilePackage(srcs *sources.Sources, tContext *types.Context) (*compiler.Archive, error) { + if archive, ok := s.UpToDateArchives[srcs.ImportPath]; ok { + return archive, nil + } + + archive, err := compiler.Compile(srcs, tContext, s.options.Minify) + if err != nil { + return nil, err + } + + for _, jsFile := range srcs.JSFiles { + archive.IncJSCode = append(archive.IncJSCode, []byte("\t(function() {\n")...) + archive.IncJSCode = append(archive.IncJSCode, jsFile.Content...) + archive.IncJSCode = append(archive.IncJSCode, []byte("\n\t}).call($global);\n")...) + } - s.UpToDateArchives[srcs.ImportPath] = archive + if s.options.Verbose { + fmt.Println(srcs.ImportPath) } - return s.UpToDateArchives[rootSrcs.ImportPath], nil + s.UpToDateArchives[srcs.ImportPath] = archive } func (s *Session) getImportPath(path, srcDir string) (string, error) { diff --git a/compiler/sources/sources.go b/compiler/sources/sources.go index d6e1fe93b..8e865bff4 100644 --- a/compiler/sources/sources.go +++ b/compiler/sources/sources.go @@ -67,9 +67,10 @@ type Importer func(path, srcDir string) (*Sources, error) // determining the type information, go linknames, etc. // // The importer function is used to import the sources of other packages -// that are imported by the package being prepared. The other sources must -// be prepared prior to being returned by the importer so that the type -// information can be used. +// that are imported by this package being prepared. If the other sources +// are not prepared when returned by the importer, that package will be +// prepared as well before continuing on with the current package. +// This is where the recursive nature of the Prepare function comes in. // // Note that at the end of this call the analysis information // has NOT been propagated across packages yet @@ -278,21 +279,21 @@ type packageImporter struct { Errors errorList.ErrorList } -func (ei *packageImporter) Import(path string) (*types.Package, error) { +func (pi *packageImporter) Import(path string) (*types.Package, error) { if path == "unsafe" { return types.Unsafe, nil } - srcs, err := ei.importer(path, ei.srcDir) + srcs, err := pi.importer(path, pi.srcDir) if err != nil { - ei.Errors = ei.Errors.AppendDistinct(err) + pi.Errors = pi.Errors.AppendDistinct(err) return nil, err } - if srcs.Package == nil { - err := srcs.Prepare(ei.importer, ei.sizes, ei.tContext) + if !srcs.isPrepared() { + err := srcs.Prepare(pi.importer, pi.sizes, pi.tContext) if err != nil { - ei.Errors = ei.Errors.AppendDistinct(err) + pi.Errors = pi.Errors.AppendDistinct(err) return nil, err } } From a1122c5bcdbc8ce0974e9c09c917dad887b5beb2 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 4 Mar 2025 13:52:07 -0700 Subject: [PATCH 17/20] Fixing remaining bugs --- build/build.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/build.go b/build/build.go index 3deb667e4..9c6313a40 100644 --- a/build/build.go +++ b/build/build.go @@ -981,6 +981,7 @@ func (s *Session) loadTestPackage(pkg *PackageData) (*sources.Sources, error) { Files: []*ast.File{mainFile}, FileSet: fset, } + s.sources[srcs.ImportPath] = srcs // Import dependencies for the testmain package. for _, importedPkgPath := range srcs.UnresolvedImports() { @@ -1159,6 +1160,7 @@ func (s *Session) compilePackage(srcs *sources.Sources, tContext *types.Context) } s.UpToDateArchives[srcs.ImportPath] = archive + return archive, nil } func (s *Session) getImportPath(path, srcDir string) (string, error) { From e57c7aabd640b328bfb9d156b55269a8f4e3441d Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 4 Mar 2025 14:22:21 -0700 Subject: [PATCH 18/20] Some cleanup --- build/build.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/build/build.go b/build/build.go index 9c6313a40..411d1aa57 100644 --- a/build/build.go +++ b/build/build.go @@ -1127,13 +1127,7 @@ func (s *Session) compilePackages(rootSrcs *sources.Sources, tContext *types.Con rootArchive, ok := s.UpToDateArchives[rootSrcs.ImportPath] if !ok { - // TODO(grantnelson-wf): Remove all but return - fmt.Printf(">> root package %q not found\n", rootSrcs.ImportPath) - fmt.Printf(">> sources contained:\n") - for _, srcs := range s.sources { - fmt.Printf(">> source: %s\n", srcs.ImportPath) - } - + // This is simply confirmation that the root package is in the sources map and got compiled. return nil, fmt.Errorf(`root package %q not found`, rootSrcs.ImportPath) } return rootArchive, nil From c3f2ffed17be9a922061c9982ebcf6ca2bd495c4 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 4 Mar 2025 17:08:14 -0700 Subject: [PATCH 19/20] Some cleanup --- build/build.go | 14 +++---- compiler/compiler_test.go | 11 +++-- compiler/internal/analysis/info.go | 4 +- compiler/internal/analysis/info_test.go | 2 +- compiler/package.go | 13 ++++-- compiler/sources/sources.go | 55 ++++++++++++++++--------- 6 files changed, 64 insertions(+), 35 deletions(-) diff --git a/build/build.go b/build/build.go index 411d1aa57..0530b648b 100644 --- a/build/build.go +++ b/build/build.go @@ -766,8 +766,7 @@ type Session struct { // up to date with input sources and dependencies. In the -w ("watch") mode // must be cleared upon entering watching. UpToDateArchives map[string]*compiler.Archive - - Watcher *fsnotify.Watcher + Watcher *fsnotify.Watcher } // NewSession creates a new GopherJS build session. @@ -949,9 +948,7 @@ func (s *Session) getSortedSources() []*sources.Sources { for _, srcs := range s.sources { allSources = append(allSources, srcs) } - sort.Slice(allSources, func(i, j int) bool { - return allSources[i].ImportPath < allSources[j].ImportPath - }) + sources.SortedSourcesSlice(allSources) return allSources } @@ -1122,12 +1119,14 @@ func (s *Session) loadPackages(pkg *PackageData) (*sources.Sources, error) { func (s *Session) compilePackages(rootSrcs *sources.Sources, tContext *types.Context) (*compiler.Archive, error) { for _, srcs := range s.sources { - s.compilePackage(srcs, tContext) + if _, err := s.compilePackage(srcs, tContext); err != nil { + return nil, err + } } rootArchive, ok := s.UpToDateArchives[rootSrcs.ImportPath] if !ok { - // This is simply confirmation that the root package is in the sources map and got compiled. + // This is confirmation that the root package is in the sources map and got compiled. return nil, fmt.Errorf(`root package %q not found`, rootSrcs.ImportPath) } return rootArchive, nil @@ -1154,6 +1153,7 @@ func (s *Session) compilePackage(srcs *sources.Sources, tContext *types.Context) } s.UpToDateArchives[srcs.ImportPath] = archive + return archive, nil } diff --git a/compiler/compiler_test.go b/compiler/compiler_test.go index d0509b4d4..2c399ad9b 100644 --- a/compiler/compiler_test.go +++ b/compiler/compiler_test.go @@ -704,9 +704,7 @@ func compileProject(t *testing.T, root *packages.Package, minify bool) map[strin for _, srcs := range allSrcs { sortedSources = append(sortedSources, srcs) } - sort.Slice(sortedSources, func(i, j int) bool { - return sortedSources[i].ImportPath < sortedSources[j].ImportPath - }) + sources.SortedSourcesSlice(sortedSources) PrepareAllSources(sortedSources, importer, tContext) archives := map[string]*Archive{} @@ -733,6 +731,13 @@ func newTime(seconds float64) time.Time { func reloadCompiledProject(t *testing.T, archives map[string]*Archive, rootPkgPath string) map[string]*Archive { t.Helper() + // TODO(grantnelson-wf): The tests using this function are out-of-date + // since they are testing the old archive caching that has been disabled. + // At some point, these tests should be updated to test any new caching + // mechanism that is implemented or removed. As is this function is faking + // the old recursive archive loading that is no longer used since it + // doesn't allow cross package analysis for generings. + buildTime := newTime(5.0) serialized := map[string][]byte{} for path, a := range archives { diff --git a/compiler/internal/analysis/info.go b/compiler/internal/analysis/info.go index f5bebc91d..246547f60 100644 --- a/compiler/internal/analysis/info.go +++ b/compiler/internal/analysis/info.go @@ -58,10 +58,12 @@ type Info struct { funcLitInfos map[*ast.FuncLit][]*FuncInfo InitFuncInfo *FuncInfo // Context for package variable initialization. - infoImporter InfoImporter // For functions from other packages. + infoImporter InfoImporter // To get `Info` for other packages. allInfos []*FuncInfo } +// InfoImporter is used to get the `Info` for another package. +// The path is the resolved import path of the package to get the `Info` for. type InfoImporter func(path string) (*Info, error) func (info *Info) newFuncInfo(n ast.Node, obj types.Object, typeArgs typesutil.TypeList, resolver *typeparams.Resolver) *FuncInfo { diff --git a/compiler/internal/analysis/info_test.go b/compiler/internal/analysis/info_test.go index e6eaf0dec..73428207e 100644 --- a/compiler/internal/analysis/info_test.go +++ b/compiler/internal/analysis/info_test.go @@ -1576,7 +1576,7 @@ func TestBlocking_IsImportBlocking_ForwardInstances(t *testing.T) { } func TestBlocking_IsImportBlocking_BackwardInstances(t *testing.T) { - // This is tests propagation of information across package boundaries. + // This tests propagation of information across package boundaries. // `FooBaz` has no instances in it until it is referenced in the `test` package. // That instance information needs to propagate back across the package // boundary to the `other` package. The information for `BazBlocker` and diff --git a/compiler/package.go b/compiler/package.go index ec1eb8634..cb06c9b1a 100644 --- a/compiler/package.go +++ b/compiler/package.go @@ -150,11 +150,18 @@ type flowData struct { endCase int } +// PrepareAllSources prepares all sources for compilation by +// parsing go linknames, type checking, sorting, simplifying, and +// performing cross package analysis. +// The results are stored in the provided sources. +// +// All sources must be given at the same time for cross package analysis to +// work correctly. For consistency, the sources should be sorted by import path. func PrepareAllSources(allSources []*sources.Sources, importer sources.Importer, tContext *types.Context) error { // This will be performed recursively for all dependencies so - // most of these prepare calls will be no-ops. Since not all packages will - // be recursively reached via the root source, we need to prepare them - // all here. + // most of these prepare calls will be no-ops. + // Since some packages might not be recursively reached via the root source, + // e.g. runtime, we need to try to prepare them all here. for _, srcs := range allSources { if err := srcs.Prepare(importer, sizes32, tContext); err != nil { return err diff --git a/compiler/sources/sources.go b/compiler/sources/sources.go index 8e865bff4..50d6f5777 100644 --- a/compiler/sources/sources.go +++ b/compiler/sources/sources.go @@ -44,19 +44,19 @@ type Sources struct { JSFiles []jsFile.JSFile // TypeInfo is the type information this package. - // This is nil until PrepareInfo is called. + // This is nil until Prepare is called. TypeInfo *analysis.Info // Instances is the type parameters instances for this package. - // This is nil until PrepareInfo is called. + // This is nil until Prepare is called. Instances *typeparams.PackageInstanceSets // Package is the type-PrepareInfo package. - // This is nil until PrepareInfo is called. + // This is nil until Prepare is called. Package *types.Package // GoLinknames is the set of Go linknames for this package. - // This is nil until PrepareInfo is called. + // This is nil until Prepare is called. GoLinknames []linkname.GoLinkname } @@ -67,14 +67,13 @@ type Importer func(path, srcDir string) (*Sources, error) // determining the type information, go linknames, etc. // // The importer function is used to import the sources of other packages -// that are imported by this package being prepared. If the other sources -// are not prepared when returned by the importer, that package will be -// prepared as well before continuing on with the current package. +// that the package being prepared depends on. If the other sources +// are not prepared when returned by the importer, then that package +// will be prepared before continuing on with the current package. // This is where the recursive nature of the Prepare function comes in. // // Note that at the end of this call the analysis information -// has NOT been propagated across packages yet -// and the source files have not been simplified yet. +// has NOT been propagated across packages yet. func (s *Sources) Prepare(importer Importer, sizes types.Sizes, tContext *types.Context) error { // Skip if the sources have already been prepared. if s.isPrepared() { @@ -113,6 +112,12 @@ func (s *Sources) Prepare(importer Importer, sizes types.Sizes, tContext *types. return nil } +// isPrepared returns true if this sources have had Prepare called on it. +// +// This can not determine if the type information has been propagated +// across packages yet, but usually would only be called prior to that. +// For the source to be fully prepared for compilation, the type information +// must be propagated across packages as well. func (s *Sources) isPrepared() bool { return s.TypeInfo != nil && s.Package != nil } @@ -132,8 +137,8 @@ func (s *Sources) sort() { // Note this function mutates the original Files slice. // This must be called after TypeCheck and before analyze since // this will change the pointers in the AST, for example the pointers -// to function literals will change making it impossible to find them -// in the type information if analyze is called first. +// to function literals will change, making it impossible to find them +// in the type information, if analyze is called first. func (s *Sources) simplify(typesInfo *types.Info) { for i, file := range s.Files { s.Files[i] = astrewrite.Simplify(file, typesInfo, false) @@ -143,7 +148,7 @@ func (s *Sources) simplify(typesInfo *types.Info) { // typeCheck the sources. Returns information about declared package types and // type information for the supplied AST. // -// This must be called prior to simplify. +// This must be called prior to simplify to get the types.Info used by simplify. func (s *Sources) typeCheck(importer Importer, sizes types.Sizes, tContext *types.Context) (*types.Info, error) { const errLimit = 10 // Max number of type checking errors to return. @@ -195,7 +200,9 @@ func (s *Sources) typeCheck(importer Importer, sizes types.Sizes, tContext *type // analyze will determine the type parameters instances, blocking, // and other type information for the package. // -// This must be called after to simplify. +// This must be called after to simplify to ensure the pointers +// in the AST are still valid. +// // Note that at the end of this call the analysis information // has NOT been propagated across packages yet. func (s *Sources) analyze(typesInfo *types.Info, importer Importer, tContext *types.Context) { @@ -222,7 +229,6 @@ func (s *Sources) analyze(typesInfo *types.Info, importer Importer, tContext *ty // parseGoLinknames extracts all //go:linkname compiler directive from the sources. // // This will set the GoLinknames field on the Sources struct. -// This must be called prior to simplify. func (s *Sources) parseGoLinknames() error { goLinknames := []linkname.GoLinkname{} var errs errorList.ErrorList @@ -290,13 +296,22 @@ func (pi *packageImporter) Import(path string) (*types.Package, error) { return nil, err } - if !srcs.isPrepared() { - err := srcs.Prepare(pi.importer, pi.sizes, pi.tContext) - if err != nil { - pi.Errors = pi.Errors.AppendDistinct(err) - return nil, err - } + // If the source hasn't been prepared yet, prepare it now. + // This will recursively prepare all of it's dependencies too. + // If the source is already prepared, this will be a no-op. + err = srcs.Prepare(pi.importer, pi.sizes, pi.tContext) + if err != nil { + pi.Errors = pi.Errors.AppendDistinct(err) + return nil, err } return srcs.Package, nil } + +// SortedSourcesSlice in place sorts the given slice of Sources by ImportPath. +// This will not change the order of the files within any Sources. +func SortedSourcesSlice(sourcesSlice []*Sources) { + sort.Slice(sourcesSlice, func(i, j int) bool { + return sourcesSlice[i].ImportPath < sourcesSlice[j].ImportPath + }) +} From aae9e0a4b428437fd9acecfeff54ed45a6e5938d Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 4 Mar 2025 17:13:07 -0700 Subject: [PATCH 20/20] Some cleanup --- build/build.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/build/build.go b/build/build.go index 0530b648b..3af3c16c4 100644 --- a/build/build.go +++ b/build/build.go @@ -759,7 +759,7 @@ type Session struct { // sources is a map of parsed packages that have been built and augmented. // This is keyed using resolved import paths. This is used to avoid // rebuilding and augmenting packages that are imported by several packages. - // These sources haven't been simplified yet. + // The files in these sources haven't been sorted nor simplified yet. sources map[string]*sources.Sources // Binary archives produced during the current session and assumed to be @@ -928,6 +928,13 @@ func (s *Session) BuildProject(pkg *PackageData) (*compiler.Archive, error) { return nil, err } + // TODO(grantnelson-wf): We could investigate caching the results of + // the sources prior to preparing them to avoid re-parsing the same + // sources and augmenting them when the files on disk haven't changed. + // This would require a way to determine if the sources are up-to-date + // which could be done with the left over srcModTime from when the archives + // were being cached. + // Prepare and analyze the source code. // This will be performed recursively for all dependencies. tContext := types.NewContext()