Skip to content

Commit 963a05a

Browse files
grantnelson-wfgrantnelson-wf
committed
separating parse and compile
1 parent 8e75705 commit 963a05a

File tree

4 files changed

+224
-98
lines changed

4 files changed

+224
-98
lines changed

‎build/build.go

Lines changed: 202 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,11 @@ import (
2727
"github.com/fsnotify/fsnotify"
2828
"github.com/gopherjs/gopherjs/compiler"
2929
"github.com/gopherjs/gopherjs/compiler/astutil"
30+
"github.com/gopherjs/gopherjs/internal/testmain"
3031
log "github.com/sirupsen/logrus"
3132

3233
"github.com/neelance/sourcemap"
3334
"golang.org/x/tools/go/buildutil"
34-
35-
"github.com/gopherjs/gopherjs/build/cache"
3635
)
3736

3837
// DefaultGOROOT is the default GOROOT value for builds.
@@ -744,14 +743,61 @@ func (p *PackageData) InstallPath() string {
744743
return p.PkgObj
745744
}
746745

746+
// ParsedPackage is the results from building a package before it is compiled.
747+
type ParsedPackage struct {
748+
ImportPath string // import path of package ("" if unknown)
749+
Dir string // directory containing package sources
750+
751+
// GoFiles is the parsed and augmented Go AST files for the package.
752+
GoFiles []*ast.File
753+
FileSet *token.FileSet
754+
JSFiles []JSFile
755+
}
756+
757+
// Imports calculates the import paths of the package's dependencies
758+
// based on all the imports in the augmented files.
759+
//
760+
// The given skip paths will not be returned in the results.
761+
// This will not return any `*_test` packages in the results.
762+
func (p *ParsedPackage) Imports(skip ...string) []string {
763+
importMap := make(map[string]struct{})
764+
for _, file := range p.GoFiles {
765+
for _, imp := range file.Imports {
766+
path := strings.Trim(imp.Path.Value, `"`)
767+
if !strings.HasSuffix(path, "_test") {
768+
importMap[path] = struct{}{}
769+
}
770+
}
771+
}
772+
for _, skip := range skip {
773+
delete(importMap, skip)
774+
}
775+
776+
imports := make([]string, 0, len(importMap))
777+
for imp := range importMap {
778+
imports = append(imports, imp)
779+
}
780+
sort.Strings(imports)
781+
return imports
782+
}
783+
747784
// Session manages internal state GopherJS requires to perform a build.
748785
//
749786
// This is the main interface to GopherJS build system. Session lifetime is
750787
// roughly equivalent to a single GopherJS tool invocation.
751788
type Session struct {
752-
options *Options
753-
xctx XContext
754-
buildCache cache.BuildCache
789+
options *Options
790+
xctx XContext
791+
792+
// importPaths is a map of the resolved import paths given the srcDir
793+
// and path. This is used to avoid redetermining build packages during
794+
// compilation when we're looking up parsed packages.
795+
importPaths map[string]map[string]string
796+
797+
// parsePackage is a map of parsed packages that have been built and augmented.
798+
// This is keyed using resolved import paths. This is used to avoid
799+
// rebuilding and augmenting packages that are imported by several packages.
800+
parsedPackages map[string]*ParsedPackage
755801

756802
// Binary archives produced during the current session and assumed to be
757803
// up to date with input sources and dependencies. In the -w ("watch") mode
@@ -767,6 +813,8 @@ func NewSession(options *Options) (*Session, error) {
767813

768814
s := &Session{
769815
options: options,
816+
importPaths: make(map[string]map[string]string),
817+
parsedPackages: make(map[string]*ParsedPackage),
770818
UpToDateArchives: make(map[string]*compiler.Archive),
771819
}
772820
s.xctx = NewBuildContext(s.InstallSuffix(), s.options.BuildTags)
@@ -777,15 +825,6 @@ func NewSession(options *Options) (*Session, error) {
777825
return nil, err
778826
}
779827

780-
s.buildCache = cache.BuildCache{
781-
GOOS: env.GOOS,
782-
GOARCH: env.GOARCH,
783-
GOROOT: env.GOROOT,
784-
GOPATH: env.GOPATH,
785-
BuildTags: append([]string{}, env.BuildTags...),
786-
Minify: options.Minify,
787-
TestedPackage: options.TestedPackage,
788-
}
789828
s.Types = make(map[string]*types.Package)
790829
if options.Watch {
791830
if out, err := exec.Command("ulimit", "-n").Output(); err == nil {
@@ -900,7 +939,7 @@ func (s *Session) BuildFiles(filenames []string, pkgObj string, cwd string) erro
900939
})
901940
}
902941

903-
archive, err := s.BuildPackage(pkg)
942+
archive, err := s.BuildProject(pkg)
904943
if err != nil {
905944
return err
906945
}
@@ -910,19 +949,76 @@ func (s *Session) BuildFiles(filenames []string, pkgObj string, cwd string) erro
910949
return s.WriteCommandPackage(archive, pkgObj)
911950
}
912951

913-
// BuildImportPath loads and compiles package with the given import path.
914-
//
915-
// Relative paths are interpreted relative to the current working dir.
916-
func (s *Session) BuildImportPath(path string) (*compiler.Archive, error) {
917-
_, archive, err := s.buildImportPathWithSrcDir(path, "")
918-
return archive, err
952+
// BuildProject builds a command project (one with a main method) or
953+
// builds a test project (one with a synthesized test main package).
954+
func (s *Session) BuildProject(pkg *PackageData) (*compiler.Archive, error) {
955+
// ensure that runtime for gopherjs is imported
956+
pkg.Imports = append(pkg.Imports, `runtime`)
957+
958+
// Build the project to get the parsed packages.
959+
var parsed *ParsedPackage
960+
var err error
961+
if pkg.IsTest {
962+
parsed, err = s.buildTestPackage(pkg)
963+
} else {
964+
parsed, err = s.buildPackages(pkg)
965+
}
966+
if err != nil {
967+
return nil, err
968+
}
969+
970+
// TODO(grantnelson-wf): At this point we have all the parsed packages we
971+
// need to compile the whole project, including testmain, if needed.
972+
// We can perform analysis on the whole project at this point to propagate
973+
// flatten, blocking, etc. information and check types to get the type info
974+
// with all the instances for all generics in the whole project.
975+
976+
return s.compilePackages(parsed)
977+
}
978+
979+
func (s *Session) buildTestPackage(pkg *PackageData) (*ParsedPackage, error) {
980+
_, err := s.buildPackages(pkg.TestPackage())
981+
if err != nil {
982+
return nil, err
983+
}
984+
_, err = s.buildPackages(pkg.XTestPackage())
985+
if err != nil {
986+
return nil, err
987+
}
988+
989+
// Generate a synthetic testmain package.
990+
fset := token.NewFileSet()
991+
tests := testmain.TestMain{Package: pkg.Package, Context: pkg.bctx}
992+
tests.Scan(fset)
993+
mainPkg, mainFile, err := tests.Synthesize(fset)
994+
if err != nil {
995+
return nil, fmt.Errorf("failed to generate testmain package for %s: %w", pkg.ImportPath, err)
996+
}
997+
998+
// Create a parsed package for the testmain package.
999+
parsed := &ParsedPackage{
1000+
ImportPath: mainPkg.ImportPath,
1001+
Dir: mainPkg.Dir,
1002+
GoFiles: []*ast.File{mainFile},
1003+
FileSet: fset,
1004+
}
1005+
1006+
// Import dependencies for the testmain package.
1007+
for _, importedPkgPath := range parsed.Imports() {
1008+
_, _, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir)
1009+
if err != nil {
1010+
return nil, err
1011+
}
1012+
}
1013+
1014+
return parsed, nil
9191015
}
9201016

921-
// buildImportPathWithSrcDir builds the package specified by the import path.
1017+
// buildImportPathWithSrcDir builds the parsed package specified by the import path.
9221018
//
9231019
// Relative import paths are interpreted relative to the passed srcDir. If
9241020
// srcDir is empty, current working directory is assumed.
925-
func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*PackageData, *compiler.Archive, error) {
1021+
func (s *Session) buildImportPathWithSrcDir(path, srcDir string) (*PackageData, *ParsedPackage, error) {
9261022
pkg, err := s.xctx.Import(path, srcDir, 0)
9271023
if s.Watcher != nil && pkg != nil { // add watch even on error
9281024
s.Watcher.Add(pkg.Dir)
@@ -931,12 +1027,25 @@ func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*Packag
9311027
return nil, nil, err
9321028
}
9331029

934-
archive, err := s.BuildPackage(pkg)
1030+
parsed, err := s.buildPackages(pkg)
9351031
if err != nil {
9361032
return nil, nil, err
9371033
}
9381034

939-
return pkg, archive, nil
1035+
s.cacheImportPath(path, srcDir, pkg.ImportPath)
1036+
return pkg, parsed, nil
1037+
}
1038+
1039+
// cacheImportPath stores the import path for the build package so we can look
1040+
// it up later without getting the whole build package.
1041+
// The given path and source directly are the ones passed into
1042+
// XContext.Import to the get the build package originally.
1043+
func (s *Session) cacheImportPath(path, srcDir, importPath string) {
1044+
if paths, ok := s.importPaths[srcDir]; ok {
1045+
paths[path] = importPath
1046+
} else {
1047+
s.importPaths[srcDir] = map[string]string{path: importPath}
1048+
}
9401049
}
9411050

9421051
// getExeModTime will determine the mod time of the GopherJS binary
@@ -965,10 +1074,9 @@ var getExeModTime = func() func() time.Time {
9651074
}
9661075
}()
9671076

968-
// BuildPackage compiles an already loaded package.
969-
func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
970-
if archive, ok := s.UpToDateArchives[pkg.ImportPath]; ok {
971-
return archive, nil
1077+
func (s *Session) buildPackages(pkg *PackageData) (*ParsedPackage, error) {
1078+
if parsed, ok := s.parsedPackages[pkg.ImportPath]; ok {
1079+
return parsed, nil
9721080
}
9731081

9741082
if exeModTime := getExeModTime(); exeModTime.After(pkg.SrcModTime) {
@@ -993,16 +1101,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
9931101
pkg.SrcModTime = fileModTime
9941102
}
9951103

996-
if !s.options.NoCache {
997-
archive := s.buildCache.LoadArchive(pkg.ImportPath, pkg.SrcModTime, s.Types)
998-
if archive != nil {
999-
s.UpToDateArchives[pkg.ImportPath] = archive
1000-
// Existing archive is up to date, no need to build it from scratch.
1001-
return archive, nil
1002-
}
1003-
}
1004-
1005-
// Existing archive is out of date or doesn't exist, let's build the package.
1104+
// Build the package by parsing and augmenting the original files with overlay files.
10061105
fileSet := token.NewFileSet()
10071106
files, overlayJsFiles, err := parseAndAugment(s.xctx, pkg, pkg.IsTest, fileSet)
10081107
if err != nil {
@@ -1016,16 +1115,42 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
10161115
files = append(files, embed)
10171116
}
10181117

1118+
parsed := &ParsedPackage{
1119+
ImportPath: pkg.ImportPath,
1120+
Dir: pkg.Dir,
1121+
GoFiles: files,
1122+
FileSet: fileSet,
1123+
JSFiles: append(pkg.JSFiles, overlayJsFiles...),
1124+
}
1125+
s.parsedPackages[pkg.ImportPath] = parsed
1126+
1127+
// Import dependencies from the augmented files,
1128+
// whilst skipping any that have been already imported.
1129+
for _, importedPkgPath := range parsed.Imports(pkg.Imports...) {
1130+
_, _, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir)
1131+
if err != nil {
1132+
return nil, err
1133+
}
1134+
}
1135+
1136+
return parsed, nil
1137+
}
1138+
1139+
func (s *Session) compilePackages(pkg *ParsedPackage) (*compiler.Archive, error) {
1140+
if archive, ok := s.UpToDateArchives[pkg.ImportPath]; ok {
1141+
return archive, nil
1142+
}
1143+
10191144
importContext := &compiler.ImportContext{
10201145
Packages: s.Types,
1021-
Import: s.ImportResolverFor(pkg),
1146+
Import: s.ImportResolverFor(pkg.Dir),
10221147
}
1023-
archive, err := compiler.Compile(pkg.ImportPath, files, fileSet, importContext, s.options.Minify)
1148+
archive, err := compiler.Compile(pkg.ImportPath, pkg.GoFiles, pkg.FileSet, importContext, s.options.Minify)
10241149
if err != nil {
10251150
return nil, err
10261151
}
10271152

1028-
for _, jsFile := range append(pkg.JSFiles, overlayJsFiles...) {
1153+
for _, jsFile := range pkg.JSFiles {
10291154
archive.IncJSCode = append(archive.IncJSCode, []byte("\t(function() {\n")...)
10301155
archive.IncJSCode = append(archive.IncJSCode, jsFile.Content...)
10311156
archive.IncJSCode = append(archive.IncJSCode, []byte("\n\t}).call($global);\n")...)
@@ -1035,21 +1160,50 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
10351160
fmt.Println(pkg.ImportPath)
10361161
}
10371162

1038-
s.buildCache.StoreArchive(archive, time.Now())
10391163
s.UpToDateArchives[pkg.ImportPath] = archive
10401164

10411165
return archive, nil
10421166
}
10431167

1168+
func (s *Session) getImportPath(path, srcDir string) (string, error) {
1169+
// If path is for an xtest package, just return it.
1170+
if strings.HasSuffix(path, "_test") {
1171+
return path, nil
1172+
}
1173+
1174+
// Check if the import path is already cached.
1175+
if importPath, ok := s.importPaths[srcDir][path]; ok {
1176+
return importPath, nil
1177+
}
1178+
1179+
// Fall back to the slop import of the build package.
1180+
pkg, err := s.xctx.Import(path, srcDir, 0)
1181+
if err != nil {
1182+
return ``, err
1183+
}
1184+
s.cacheImportPath(path, srcDir, pkg.ImportPath)
1185+
return pkg.ImportPath, nil
1186+
}
1187+
10441188
// ImportResolverFor returns a function which returns a compiled package archive
10451189
// given an import path.
1046-
func (s *Session) ImportResolverFor(pkg *PackageData) func(string) (*compiler.Archive, error) {
1190+
func (s *Session) ImportResolverFor(srcDir string) func(string) (*compiler.Archive, error) {
10471191
return func(path string) (*compiler.Archive, error) {
1048-
if archive, ok := s.UpToDateArchives[path]; ok {
1192+
importPath, err := s.getImportPath(path, srcDir)
1193+
if err != nil {
1194+
return nil, err
1195+
}
1196+
1197+
if archive, ok := s.UpToDateArchives[importPath]; ok {
10491198
return archive, nil
10501199
}
1051-
_, archive, err := s.buildImportPathWithSrcDir(path, pkg.Dir)
1052-
return archive, err
1200+
1201+
// The archive hasn't been compiled yet so compile it with the parsed package.
1202+
if parsed, ok := s.parsedPackages[importPath]; ok {
1203+
return s.compilePackages(parsed)
1204+
}
1205+
1206+
return nil, fmt.Errorf(`parsed package for %q not found`, importPath)
10531207
}
10541208
}
10551209

@@ -1087,13 +1241,7 @@ func (s *Session) WriteCommandPackage(archive *compiler.Archive, pkgObj string)
10871241
sourceMapFilter.MappingCallback = s.SourceMappingCallback(m)
10881242
}
10891243

1090-
deps, err := compiler.ImportDependencies(archive, func(path string) (*compiler.Archive, error) {
1091-
if archive, ok := s.UpToDateArchives[path]; ok {
1092-
return archive, nil
1093-
}
1094-
_, archive, err := s.buildImportPathWithSrcDir(path, "")
1095-
return archive, err
1096-
})
1244+
deps, err := compiler.ImportDependencies(archive, s.ImportResolverFor(""))
10971245
if err != nil {
10981246
return err
10991247
}

0 commit comments

Comments
 (0)