Skip to content

Commit 887f51e

Browse files
grantnelson-wfgrantnelson-wf
committed
Updated to cross package analysis
1 parent bd9aac3 commit 887f51e

File tree

8 files changed

+509
-232
lines changed

8 files changed

+509
-232
lines changed

‎build/build.go

Lines changed: 70 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -759,14 +759,13 @@ type Session struct {
759759
// sources is a map of parsed packages that have been built and augmented.
760760
// This is keyed using resolved import paths. This is used to avoid
761761
// rebuilding and augmenting packages that are imported by several packages.
762-
// These sources haven't been sorted nor simplified yet.
762+
// The files in these sources haven't been sorted nor simplified yet.
763763
sources map[string]*sources.Sources
764764

765765
// Binary archives produced during the current session and assumed to be
766766
// up to date with input sources and dependencies. In the -w ("watch") mode
767767
// must be cleared upon entering watching.
768768
UpToDateArchives map[string]*compiler.Archive
769-
Types map[string]*types.Package
770769
Watcher *fsnotify.Watcher
771770
}
772771

@@ -788,7 +787,6 @@ func NewSession(options *Options) (*Session, error) {
788787
return nil, err
789788
}
790789

791-
s.Types = make(map[string]*types.Package)
792790
if options.Watch {
793791
if out, err := exec.Command("ulimit", "-n").Output(); err == nil {
794792
if n, err := strconv.Atoi(strings.TrimSpace(string(out))); err == nil && n < 1024 {
@@ -906,7 +904,7 @@ func (s *Session) BuildFiles(filenames []string, pkgObj string, cwd string) erro
906904
if err != nil {
907905
return err
908906
}
909-
if s.Types["main"].Name() != "main" {
907+
if s.sources["main"].Package.Name() != "main" {
910908
return fmt.Errorf("cannot build/run non-main package")
911909
}
912910
return s.WriteCommandPackage(archive, pkgObj)
@@ -918,25 +916,38 @@ func (s *Session) BuildProject(pkg *PackageData) (*compiler.Archive, error) {
918916
// ensure that runtime for gopherjs is imported
919917
pkg.Imports = append(pkg.Imports, `runtime`)
920918

921-
// Build the project to get the sources for the parsed packages.
922-
var srcs *sources.Sources
919+
// Load the project to get the sources for the parsed packages.
920+
var rootSrcs *sources.Sources
923921
var err error
924922
if pkg.IsTest {
925-
srcs, err = s.loadTestPackage(pkg)
923+
rootSrcs, err = s.loadTestPackage(pkg)
926924
} else {
927-
srcs, err = s.loadPackages(pkg)
925+
rootSrcs, err = s.loadPackages(pkg)
928926
}
929927
if err != nil {
930928
return nil, err
931929
}
932930

933-
// TODO(grantnelson-wf): At this point we have all the parsed packages we
934-
// need to compile the whole project, including testmain, if needed.
935-
// We can perform analysis on the whole project at this point to propagate
936-
// flatten, blocking, etc. information and check types to get the type info
937-
// with all the instances for all generics in the whole project.
931+
// TODO(grantnelson-wf): We could investigate caching the results of
932+
// the sources prior to preparing them to avoid re-parsing the same
933+
// sources and augmenting them when the files on disk haven't changed.
934+
// This would require a way to determine if the sources are up-to-date
935+
// which could be done with the left over srcModTime from when the archives
936+
// were being cached.
938937

939-
return s.compilePackages(srcs)
938+
// Compile the project into Archives containing the generated JS.
939+
return s.prepareAndCompilePackages(rootSrcs)
940+
}
941+
942+
// getSortedSources returns the sources sorted by import path.
943+
// The files in the sources may still not be sorted yet.
944+
func (s *Session) getSortedSources() []*sources.Sources {
945+
allSources := make([]*sources.Sources, 0, len(s.sources))
946+
for _, srcs := range s.sources {
947+
allSources = append(allSources, srcs)
948+
}
949+
sources.SortedSourcesSlice(allSources)
950+
return allSources
940951
}
941952

942953
func (s *Session) loadTestPackage(pkg *PackageData) (*sources.Sources, error) {
@@ -965,6 +976,7 @@ func (s *Session) loadTestPackage(pkg *PackageData) (*sources.Sources, error) {
965976
Files: []*ast.File{mainFile},
966977
FileSet: fset,
967978
}
979+
s.sources[srcs.ImportPath] = srcs
968980

969981
// Import dependencies for the testmain package.
970982
for _, importedPkgPath := range srcs.UnresolvedImports() {
@@ -1103,16 +1115,37 @@ func (s *Session) loadPackages(pkg *PackageData) (*sources.Sources, error) {
11031115
return srcs, nil
11041116
}
11051117

1106-
func (s *Session) compilePackages(srcs *sources.Sources) (*compiler.Archive, error) {
1118+
func (s *Session) prepareAndCompilePackages(rootSrcs *sources.Sources) (*compiler.Archive, error) {
1119+
tContext := types.NewContext()
1120+
allSources := s.getSortedSources()
1121+
1122+
// Prepare and analyze the source code.
1123+
// This will be performed recursively for all dependencies.
1124+
if err := compiler.PrepareAllSources(allSources, s.SourcesForImport, tContext); err != nil {
1125+
return nil, err
1126+
}
1127+
1128+
// Compile all the sources into archives.
1129+
for _, srcs := range allSources {
1130+
if _, err := s.compilePackage(srcs, tContext); err != nil {
1131+
return nil, err
1132+
}
1133+
}
1134+
1135+
rootArchive, ok := s.UpToDateArchives[rootSrcs.ImportPath]
1136+
if !ok {
1137+
// This is confirmation that the root package is in the sources map and got compiled.
1138+
return nil, fmt.Errorf(`root package %q was not found in archives`, rootSrcs.ImportPath)
1139+
}
1140+
return rootArchive, nil
1141+
}
1142+
1143+
func (s *Session) compilePackage(srcs *sources.Sources, tContext *types.Context) (*compiler.Archive, error) {
11071144
if archive, ok := s.UpToDateArchives[srcs.ImportPath]; ok {
11081145
return archive, nil
11091146
}
11101147

1111-
importContext := &compiler.ImportContext{
1112-
Packages: s.Types,
1113-
ImportArchive: s.ImportResolverFor(srcs.Dir),
1114-
}
1115-
archive, err := compiler.Compile(*srcs, importContext, s.options.Minify)
1148+
archive, err := compiler.Compile(srcs, tContext, s.options.Minify)
11161149
if err != nil {
11171150
return nil, err
11181151
}
@@ -1152,6 +1185,20 @@ func (s *Session) getImportPath(path, srcDir string) (string, error) {
11521185
return pkg.ImportPath, nil
11531186
}
11541187

1188+
func (s *Session) SourcesForImport(path, srcDir string) (*sources.Sources, error) {
1189+
importPath, err := s.getImportPath(path, srcDir)
1190+
if err != nil {
1191+
return nil, err
1192+
}
1193+
1194+
srcs, ok := s.sources[importPath]
1195+
if !ok {
1196+
return nil, fmt.Errorf(`sources for %q not found`, path)
1197+
}
1198+
1199+
return srcs, nil
1200+
}
1201+
11551202
// ImportResolverFor returns a function which returns a compiled package archive
11561203
// given an import path.
11571204
func (s *Session) ImportResolverFor(srcDir string) func(string) (*compiler.Archive, error) {
@@ -1165,12 +1212,7 @@ func (s *Session) ImportResolverFor(srcDir string) func(string) (*compiler.Archi
11651212
return archive, nil
11661213
}
11671214

1168-
// The archive hasn't been compiled yet so compile it with the sources.
1169-
if srcs, ok := s.sources[importPath]; ok {
1170-
return s.compilePackages(srcs)
1171-
}
1172-
1173-
return nil, fmt.Errorf(`sources for %q not found`, importPath)
1215+
return nil, fmt.Errorf(`archive for %q not found`, importPath)
11741216
}
11751217
}
11761218

@@ -1258,8 +1300,9 @@ func hasGopathPrefix(file, gopath string) (hasGopathPrefix bool, prefixLen int)
12581300
func (s *Session) WaitForChange() {
12591301
// Will need to re-validate up-to-dateness of all archives, so flush them from
12601302
// memory.
1303+
s.importPaths = map[string]map[string]string{}
1304+
s.sources = map[string]*sources.Sources{}
12611305
s.UpToDateArchives = map[string]*compiler.Archive{}
1262-
s.Types = map[string]*types.Package{}
12631306

12641307
s.options.PrintSuccess("watching for changes...\n")
12651308
for {

‎compiler/compiler_test.go

Lines changed: 153 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -613,6 +613,115 @@ func TestDeclNaming_VarsAndTypes(t *testing.T) {
613613
)
614614
}
615615

616+
func Test_CrossPackageAnalysis(t *testing.T) {
617+
src1 := `
618+
package main
619+
import "github.com/gopherjs/gopherjs/compiler/stable"
620+
621+
func main() {
622+
m := map[string]int{
623+
"one": 1,
624+
"two": 2,
625+
"three": 3,
626+
}
627+
stable.Print(m)
628+
}`
629+
src2 := `
630+
package collections
631+
import "github.com/gopherjs/gopherjs/compiler/cmp"
632+
633+
func Keys[K cmp.Ordered, V any, M ~map[K]V](m M) []K {
634+
keys := make([]K, 0, len(m))
635+
for k := range m {
636+
keys = append(keys, k)
637+
}
638+
return keys
639+
}`
640+
src3 := `
641+
package collections
642+
import "github.com/gopherjs/gopherjs/compiler/cmp"
643+
644+
func Values[K cmp.Ordered, V any, M ~map[K]V](m M) []V {
645+
values := make([]V, 0, len(m))
646+
for _, v := range m {
647+
values = append(values, v)
648+
}
649+
return values
650+
}`
651+
src4 := `
652+
package sorts
653+
import "github.com/gopherjs/gopherjs/compiler/cmp"
654+
655+
func Pair[K cmp.Ordered, V any, SK ~[]K, SV ~[]V](k SK, v SV) {
656+
Bubble(len(k),
657+
func(i, j int) bool { return k[i] < k[j] },
658+
func(i, j int) { k[i], v[i], k[j], v[j] = k[j], v[j], k[i], v[i] })
659+
}
660+
661+
func Bubble(length int, less func(i, j int) bool, swap func(i, j int)) {
662+
for i := 0; i < length; i++ {
663+
for j := i + 1; j < length; j++ {
664+
if less(j, i) {
665+
swap(i, j)
666+
}
667+
}
668+
}
669+
}`
670+
src5 := `
671+
package stable
672+
import (
673+
"github.com/gopherjs/gopherjs/compiler/collections"
674+
"github.com/gopherjs/gopherjs/compiler/sorts"
675+
"github.com/gopherjs/gopherjs/compiler/cmp"
676+
)
677+
678+
func Print[K cmp.Ordered, V any, M ~map[K]V](m M) {
679+
keys := collections.Keys(m)
680+
values := collections.Values(m)
681+
sorts.Pair(keys, values)
682+
for i, k := range keys {
683+
println(i, k, values[i])
684+
}
685+
}`
686+
src6 := `
687+
package cmp
688+
type Ordered interface { ~int | ~uint | ~float64 | ~string }`
689+
690+
root := srctesting.ParseSources(t,
691+
[]srctesting.Source{
692+
{Name: `main.go`, Contents: []byte(src1)},
693+
},
694+
[]srctesting.Source{
695+
{Name: `collections/keys.go`, Contents: []byte(src2)},
696+
{Name: `collections/values.go`, Contents: []byte(src3)},
697+
{Name: `sorts/sorts.go`, Contents: []byte(src4)},
698+
{Name: `stable/print.go`, Contents: []byte(src5)},
699+
{Name: `cmp/ordered.go`, Contents: []byte(src6)},
700+
})
701+
702+
archives := compileProject(t, root, false)
703+
checkForDeclFullNames(t, archives,
704+
// collections
705+
`funcVar:github.com/gopherjs/gopherjs/compiler/collections.Values`,
706+
`func:github.com/gopherjs/gopherjs/compiler/collections.Values<string, int, map[string]int>`,
707+
`funcVar:github.com/gopherjs/gopherjs/compiler/collections.Keys`,
708+
`func:github.com/gopherjs/gopherjs/compiler/collections.Keys<string, int, map[string]int>`,
709+
710+
// sorts
711+
`funcVar:github.com/gopherjs/gopherjs/compiler/sorts.Pair`,
712+
`func:github.com/gopherjs/gopherjs/compiler/sorts.Pair<string, int, []string, []int>`,
713+
`funcVar:github.com/gopherjs/gopherjs/compiler/sorts.Bubble`,
714+
`func:github.com/gopherjs/gopherjs/compiler/sorts.Bubble`,
715+
716+
// stable
717+
`funcVar:github.com/gopherjs/gopherjs/compiler/stable.Print`,
718+
`func:github.com/gopherjs/gopherjs/compiler/stable.Print<string, int, map[string]int>`,
719+
720+
// main
721+
`init:main`,
722+
)
723+
}
724+
616725
func TestArchiveSelectionAfterSerialization(t *testing.T) {
617726
src := `
618727
package main
@@ -679,43 +788,43 @@ func compileProject(t *testing.T, root *packages.Package, minify bool) map[strin
679788
pkgMap[pkg.PkgPath] = pkg
680789
})
681790

682-
archiveCache := map[string]*Archive{}
683-
var importContext *ImportContext
684-
importContext = &ImportContext{
685-
Packages: map[string]*types.Package{},
686-
ImportArchive: func(path string) (*Archive, error) {
687-
// find in local cache
688-
if a, ok := archiveCache[path]; ok {
689-
return a, nil
690-
}
691-
692-
pkg, ok := pkgMap[path]
693-
if !ok {
694-
t.Fatal(`package not found:`, path)
695-
}
696-
importContext.Packages[path] = pkg.Types
697-
698-
srcs := sources.Sources{
699-
ImportPath: path,
700-
Files: pkg.Syntax,
701-
FileSet: pkg.Fset,
702-
}
791+
allSrcs := map[string]*sources.Sources{}
792+
for _, pkg := range pkgMap {
793+
srcs := &sources.Sources{
794+
ImportPath: pkg.PkgPath,
795+
Dir: ``,
796+
Files: pkg.Syntax,
797+
FileSet: pkg.Fset,
798+
}
799+
allSrcs[pkg.PkgPath] = srcs
800+
}
703801

704-
// compile package
705-
a, err := Compile(srcs, importContext, minify)
706-
if err != nil {
707-
return nil, err
708-
}
709-
archiveCache[path] = a
710-
return a, nil
711-
},
802+
importer := func(path, srcDir string) (*sources.Sources, error) {
803+
srcs, ok := allSrcs[path]
804+
if !ok {
805+
t.Fatal(`package not found:`, path)
806+
return nil, nil
807+
}
808+
return srcs, nil
712809
}
713810

714-
_, err := importContext.ImportArchive(root.PkgPath)
715-
if err != nil {
716-
t.Fatal(`failed to compile:`, err)
811+
tContext := types.NewContext()
812+
sortedSources := make([]*sources.Sources, 0, len(allSrcs))
813+
for _, srcs := range allSrcs {
814+
sortedSources = append(sortedSources, srcs)
717815
}
718-
return archiveCache
816+
sources.SortedSourcesSlice(sortedSources)
817+
PrepareAllSources(sortedSources, importer, tContext)
818+
819+
archives := map[string]*Archive{}
820+
for _, srcs := range allSrcs {
821+
a, err := Compile(srcs, tContext, minify)
822+
if err != nil {
823+
t.Fatal(`failed to compile:`, err)
824+
}
825+
archives[srcs.ImportPath] = a
826+
}
827+
return archives
719828
}
720829

721830
// newTime creates an arbitrary time.Time offset by the given number of seconds.
@@ -730,6 +839,13 @@ func newTime(seconds float64) time.Time {
730839
func reloadCompiledProject(t *testing.T, archives map[string]*Archive, rootPkgPath string) map[string]*Archive {
731840
t.Helper()
732841

842+
// TODO(grantnelson-wf): The tests using this function are out-of-date
843+
// since they are testing the old archive caching that has been disabled.
844+
// At some point, these tests should be updated to test any new caching
845+
// mechanism that is implemented or removed. As is this function is faking
846+
// the old recursive archive loading that is no longer used since it
847+
// doesn't allow cross package analysis for generings.
848+
733849
buildTime := newTime(5.0)
734850
serialized := map[string][]byte{}
735851
for path, a := range archives {
@@ -742,6 +858,10 @@ func reloadCompiledProject(t *testing.T, archives map[string]*Archive, rootPkgPa
742858

743859
srcModTime := newTime(0.0)
744860
reloadCache := map[string]*Archive{}
861+
type ImportContext struct {
862+
Packages map[string]*types.Package
863+
ImportArchive func(path string) (*Archive, error)
864+
}
745865
var importContext *ImportContext
746866
importContext = &ImportContext{
747867
Packages: map[string]*types.Package{},

0 commit comments

Comments
 (0)