Skip to content

Commit bd9aac3

Browse files
grantnelson-wfgrantnelson-wf
authored
Merge pull request #1362 from Workiva/sourceInsteadOfParsedPackage
Use sources instead of parsed package
2 parents a3f0ab4 + c8b8905 commit bd9aac3

File tree

14 files changed

+357
-296
lines changed

14 files changed

+357
-296
lines changed

‎build/build.go

Lines changed: 50 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ import (
2727
"github.com/fsnotify/fsnotify"
2828
"github.com/gopherjs/gopherjs/compiler"
2929
"github.com/gopherjs/gopherjs/compiler/astutil"
30+
"github.com/gopherjs/gopherjs/compiler/jsFile"
31+
"github.com/gopherjs/gopherjs/compiler/sources"
32+
"github.com/gopherjs/gopherjs/internal/errorList"
3033
"github.com/gopherjs/gopherjs/internal/testmain"
3134
log "github.com/sirupsen/logrus"
3235

@@ -163,7 +166,7 @@ type overrideInfo struct {
163166
// - Otherwise for identifiers that exist in the original and the overrides,
164167
// the original is removed.
165168
// - New identifiers that don't exist in original package get added.
166-
func parseAndAugment(xctx XContext, pkg *PackageData, isTest bool, fileSet *token.FileSet) ([]*ast.File, []JSFile, error) {
169+
func parseAndAugment(xctx XContext, pkg *PackageData, isTest bool, fileSet *token.FileSet) ([]*ast.File, []jsFile.JSFile, error) {
167170
jsFiles, overlayFiles := parseOverlayFiles(xctx, pkg, isTest, fileSet)
168171

169172
originalFiles, err := parserOriginalFiles(pkg, fileSet)
@@ -192,7 +195,7 @@ func parseAndAugment(xctx XContext, pkg *PackageData, isTest bool, fileSet *toke
192195

193196
// parseOverlayFiles loads and parses overlay files
194197
// to augment the original files with.
195-
func parseOverlayFiles(xctx XContext, pkg *PackageData, isTest bool, fileSet *token.FileSet) ([]JSFile, []*ast.File) {
198+
func parseOverlayFiles(xctx XContext, pkg *PackageData, isTest bool, fileSet *token.FileSet) ([]jsFile.JSFile, []*ast.File) {
196199
isXTest := strings.HasSuffix(pkg.ImportPath, "_test")
197200
importPath := pkg.ImportPath
198201
if isXTest {
@@ -238,7 +241,7 @@ func parseOverlayFiles(xctx XContext, pkg *PackageData, isTest bool, fileSet *to
238241
// parserOriginalFiles loads and parses the original files to augment.
239242
func parserOriginalFiles(pkg *PackageData, fileSet *token.FileSet) ([]*ast.File, error) {
240243
var files []*ast.File
241-
var errList compiler.ErrorList
244+
var errList errorList.ErrorList
242245
for _, name := range pkg.GoFiles {
243246
if !filepath.IsAbs(name) { // name might be absolute if specified directly. E.g., `gopherjs build /abs/file.go`.
244247
name = filepath.Join(pkg.Dir, name)
@@ -622,18 +625,11 @@ func (o *Options) PrintSuccess(format string, a ...interface{}) {
622625
fmt.Fprintf(os.Stderr, format, a...)
623626
}
624627

625-
// JSFile represents a *.inc.js file metadata and content.
626-
type JSFile struct {
627-
Path string // Full file path for the build context the file came from.
628-
ModTime time.Time
629-
Content []byte
630-
}
631-
632628
// PackageData is an extension of go/build.Package with additional metadata
633629
// GopherJS requires.
634630
type PackageData struct {
635631
*build.Package
636-
JSFiles []JSFile
632+
JSFiles []jsFile.JSFile
637633
// IsTest is true if the package is being built for running tests.
638634
IsTest bool
639635
SrcModTime time.Time
@@ -743,43 +739,6 @@ func (p *PackageData) InstallPath() string {
743739
return p.PkgObj
744740
}
745741

746-
// ParsedPackage is the results from building a package before it is compiled.
747-
type ParsedPackage struct {
748-
ImportPath string // import path of package ("" if unknown)
749-
Dir string // directory containing package sources
750-
751-
// GoFiles is the parsed and augmented Go AST files for the package.
752-
GoFiles []*ast.File
753-
FileSet *token.FileSet
754-
JSFiles []JSFile
755-
}
756-
757-
// Imports calculates the import paths of the package's dependencies
758-
// based on all the imports in the augmented files.
759-
//
760-
// The given skip paths will not be returned in the results.
761-
// This will not return any `*_test` packages in the results.
762-
func (p *ParsedPackage) Imports(skip ...string) []string {
763-
seen := make(map[string]struct{})
764-
for _, s := range skip {
765-
seen[s] = struct{}{}
766-
}
767-
imports := []string{}
768-
for _, file := range p.GoFiles {
769-
for _, imp := range file.Imports {
770-
path := strings.Trim(imp.Path.Value, `"`)
771-
if _, ok := seen[path]; !ok {
772-
if !strings.HasSuffix(path, "_test") {
773-
imports = append(imports, path)
774-
}
775-
seen[path] = struct{}{}
776-
}
777-
}
778-
}
779-
sort.Strings(imports)
780-
return imports
781-
}
782-
783742
// Session manages internal state GopherJS requires to perform a build.
784743
//
785744
// This is the main interface to GopherJS build system. Session lifetime is
@@ -797,10 +756,11 @@ type Session struct {
797756
// is the unresolved import path and source directory.
798757
importPaths map[string]map[string]string
799758

800-
// parsePackage is a map of parsed packages that have been built and augmented.
759+
// sources is a map of parsed packages that have been built and augmented.
801760
// This is keyed using resolved import paths. This is used to avoid
802761
// rebuilding and augmenting packages that are imported by several packages.
803-
parsedPackages map[string]*ParsedPackage
762+
// These sources haven't been sorted nor simplified yet.
763+
sources map[string]*sources.Sources
804764

805765
// Binary archives produced during the current session and assumed to be
806766
// up to date with input sources and dependencies. In the -w ("watch") mode
@@ -817,7 +777,7 @@ func NewSession(options *Options) (*Session, error) {
817777
s := &Session{
818778
options: options,
819779
importPaths: make(map[string]map[string]string),
820-
parsedPackages: make(map[string]*ParsedPackage),
780+
sources: make(map[string]*sources.Sources),
821781
UpToDateArchives: make(map[string]*compiler.Archive),
822782
}
823783
s.xctx = NewBuildContext(s.InstallSuffix(), s.options.BuildTags)
@@ -935,7 +895,7 @@ func (s *Session) BuildFiles(filenames []string, pkgObj string, cwd string) erro
935895
if err != nil {
936896
return fmt.Errorf("failed to stat %s: %w", file, err)
937897
}
938-
pkg.JSFiles = append(pkg.JSFiles, JSFile{
898+
pkg.JSFiles = append(pkg.JSFiles, jsFile.JSFile{
939899
Path: filepath.Join(pkg.Dir, filepath.Base(file)),
940900
ModTime: info.ModTime(),
941901
Content: content,
@@ -958,13 +918,13 @@ func (s *Session) BuildProject(pkg *PackageData) (*compiler.Archive, error) {
958918
// ensure that runtime for gopherjs is imported
959919
pkg.Imports = append(pkg.Imports, `runtime`)
960920

961-
// Build the project to get the parsed packages.
962-
var parsed *ParsedPackage
921+
// Build the project to get the sources for the parsed packages.
922+
var srcs *sources.Sources
963923
var err error
964924
if pkg.IsTest {
965-
parsed, err = s.loadTestPackage(pkg)
925+
srcs, err = s.loadTestPackage(pkg)
966926
} else {
967-
parsed, err = s.loadPackages(pkg)
927+
srcs, err = s.loadPackages(pkg)
968928
}
969929
if err != nil {
970930
return nil, err
@@ -976,10 +936,10 @@ func (s *Session) BuildProject(pkg *PackageData) (*compiler.Archive, error) {
976936
// flatten, blocking, etc. information and check types to get the type info
977937
// with all the instances for all generics in the whole project.
978938

979-
return s.compilePackages(parsed)
939+
return s.compilePackages(srcs)
980940
}
981941

982-
func (s *Session) loadTestPackage(pkg *PackageData) (*ParsedPackage, error) {
942+
func (s *Session) loadTestPackage(pkg *PackageData) (*sources.Sources, error) {
983943
_, err := s.loadPackages(pkg.TestPackage())
984944
if err != nil {
985945
return nil, err
@@ -998,30 +958,30 @@ func (s *Session) loadTestPackage(pkg *PackageData) (*ParsedPackage, error) {
998958
return nil, fmt.Errorf("failed to generate testmain package for %s: %w", pkg.ImportPath, err)
999959
}
1000960

1001-
// Create a parsed package for the testmain package.
1002-
parsed := &ParsedPackage{
961+
// Create the sources for parsed package for the testmain package.
962+
srcs := &sources.Sources{
1003963
ImportPath: mainPkg.ImportPath,
1004964
Dir: mainPkg.Dir,
1005-
GoFiles: []*ast.File{mainFile},
965+
Files: []*ast.File{mainFile},
1006966
FileSet: fset,
1007967
}
1008968

1009969
// Import dependencies for the testmain package.
1010-
for _, importedPkgPath := range parsed.Imports() {
970+
for _, importedPkgPath := range srcs.UnresolvedImports() {
1011971
_, _, err := s.loadImportPathWithSrcDir(importedPkgPath, pkg.Dir)
1012972
if err != nil {
1013973
return nil, err
1014974
}
1015975
}
1016976

1017-
return parsed, nil
977+
return srcs, nil
1018978
}
1019979

1020980
// loadImportPathWithSrcDir gets the parsed package specified by the import path.
1021981
//
1022982
// Relative import paths are interpreted relative to the passed srcDir.
1023983
// If srcDir is empty, current working directory is assumed.
1024-
func (s *Session) loadImportPathWithSrcDir(path, srcDir string) (*PackageData, *ParsedPackage, error) {
984+
func (s *Session) loadImportPathWithSrcDir(path, srcDir string) (*PackageData, *sources.Sources, error) {
1025985
pkg, err := s.xctx.Import(path, srcDir, 0)
1026986
if s.Watcher != nil && pkg != nil { // add watch even on error
1027987
s.Watcher.Add(pkg.Dir)
@@ -1030,13 +990,13 @@ func (s *Session) loadImportPathWithSrcDir(path, srcDir string) (*PackageData, *
1030990
return nil, nil, err
1031991
}
1032992

1033-
parsed, err := s.loadPackages(pkg)
993+
srcs, err := s.loadPackages(pkg)
1034994
if err != nil {
1035995
return nil, nil, err
1036996
}
1037997

1038998
s.cacheImportPath(path, srcDir, pkg.ImportPath)
1039-
return pkg, parsed, nil
999+
return pkg, srcs, nil
10401000
}
10411001

10421002
// cacheImportPath stores the resolved import path for the build package
@@ -1077,9 +1037,13 @@ var getExeModTime = func() func() time.Time {
10771037
}
10781038
}()
10791039

1080-
func (s *Session) loadPackages(pkg *PackageData) (*ParsedPackage, error) {
1081-
if parsed, ok := s.parsedPackages[pkg.ImportPath]; ok {
1082-
return parsed, nil
1040+
// loadPackages will recursively load and parse the given package and
1041+
// its dependencies. This will return the sources for the given package.
1042+
// The returned source and sources for the dependencies will be added
1043+
// to the session's sources map.
1044+
func (s *Session) loadPackages(pkg *PackageData) (*sources.Sources, error) {
1045+
if srcs, ok := s.sources[pkg.ImportPath]; ok {
1046+
return srcs, nil
10831047
}
10841048

10851049
if exeModTime := getExeModTime(); exeModTime.After(pkg.SrcModTime) {
@@ -1118,52 +1082,52 @@ func (s *Session) loadPackages(pkg *PackageData) (*ParsedPackage, error) {
11181082
files = append(files, embed)
11191083
}
11201084

1121-
parsed := &ParsedPackage{
1085+
srcs := &sources.Sources{
11221086
ImportPath: pkg.ImportPath,
11231087
Dir: pkg.Dir,
1124-
GoFiles: files,
1088+
Files: files,
11251089
FileSet: fileSet,
11261090
JSFiles: append(pkg.JSFiles, overlayJsFiles...),
11271091
}
1128-
s.parsedPackages[pkg.ImportPath] = parsed
1092+
s.sources[pkg.ImportPath] = srcs
11291093

11301094
// Import dependencies from the augmented files,
11311095
// whilst skipping any that have been already imported.
1132-
for _, importedPkgPath := range parsed.Imports(pkg.Imports...) {
1096+
for _, importedPkgPath := range srcs.UnresolvedImports(pkg.Imports...) {
11331097
_, _, err := s.loadImportPathWithSrcDir(importedPkgPath, pkg.Dir)
11341098
if err != nil {
11351099
return nil, err
11361100
}
11371101
}
11381102

1139-
return parsed, nil
1103+
return srcs, nil
11401104
}
11411105

1142-
func (s *Session) compilePackages(pkg *ParsedPackage) (*compiler.Archive, error) {
1143-
if archive, ok := s.UpToDateArchives[pkg.ImportPath]; ok {
1106+
func (s *Session) compilePackages(srcs *sources.Sources) (*compiler.Archive, error) {
1107+
if archive, ok := s.UpToDateArchives[srcs.ImportPath]; ok {
11441108
return archive, nil
11451109
}
11461110

11471111
importContext := &compiler.ImportContext{
1148-
Packages: s.Types,
1149-
Import: s.ImportResolverFor(pkg.Dir),
1112+
Packages: s.Types,
1113+
ImportArchive: s.ImportResolverFor(srcs.Dir),
11501114
}
1151-
archive, err := compiler.Compile(pkg.ImportPath, pkg.GoFiles, pkg.FileSet, importContext, s.options.Minify)
1115+
archive, err := compiler.Compile(*srcs, importContext, s.options.Minify)
11521116
if err != nil {
11531117
return nil, err
11541118
}
11551119

1156-
for _, jsFile := range pkg.JSFiles {
1120+
for _, jsFile := range srcs.JSFiles {
11571121
archive.IncJSCode = append(archive.IncJSCode, []byte("\t(function() {\n")...)
11581122
archive.IncJSCode = append(archive.IncJSCode, jsFile.Content...)
11591123
archive.IncJSCode = append(archive.IncJSCode, []byte("\n\t}).call($global);\n")...)
11601124
}
11611125

11621126
if s.options.Verbose {
1163-
fmt.Println(pkg.ImportPath)
1127+
fmt.Println(srcs.ImportPath)
11641128
}
11651129

1166-
s.UpToDateArchives[pkg.ImportPath] = archive
1130+
s.UpToDateArchives[srcs.ImportPath] = archive
11671131

11681132
return archive, nil
11691133
}
@@ -1201,12 +1165,12 @@ func (s *Session) ImportResolverFor(srcDir string) func(string) (*compiler.Archi
12011165
return archive, nil
12021166
}
12031167

1204-
// The archive hasn't been compiled yet so compile it with the parsed package.
1205-
if parsed, ok := s.parsedPackages[importPath]; ok {
1206-
return s.compilePackages(parsed)
1168+
// The archive hasn't been compiled yet so compile it with the sources.
1169+
if srcs, ok := s.sources[importPath]; ok {
1170+
return s.compilePackages(srcs)
12071171
}
12081172

1209-
return nil, fmt.Errorf(`parsed package for %q not found`, importPath)
1173+
return nil, fmt.Errorf(`sources for %q not found`, importPath)
12101174
}
12111175
}
12121176

‎build/context.go

Lines changed: 2 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import (
44
"fmt"
55
"go/build"
66
"go/token"
7-
"io"
87
"net/http"
98
"os"
109
"os/exec"
@@ -16,6 +15,7 @@ import (
1615
_ "github.com/gopherjs/gopherjs/build/versionhack" // go/build release tags hack.
1716
"github.com/gopherjs/gopherjs/compiler"
1817
"github.com/gopherjs/gopherjs/compiler/gopherjspkg"
18+
"github.com/gopherjs/gopherjs/compiler/jsFile"
1919
"github.com/gopherjs/gopherjs/compiler/natives"
2020
"golang.org/x/tools/go/buildutil"
2121
)
@@ -91,7 +91,7 @@ func (sc simpleCtx) Import(importPath string, srcDir string, mode build.ImportMo
9191
if err != nil {
9292
return nil, err
9393
}
94-
jsFiles, err := jsFilesFromDir(&sc.bctx, pkg.Dir)
94+
jsFiles, err := jsFile.JSFilesFromDir(&sc.bctx, pkg.Dir)
9595
if err != nil {
9696
return nil, fmt.Errorf("failed to enumerate .inc.js files in %s: %w", pkg.Dir, err)
9797
}
@@ -440,40 +440,3 @@ func updateImports(sources []string, importPos map[string][]token.Position) (new
440440
sort.Strings(newImports)
441441
return newImports, newImportPos
442442
}
443-
444-
// jsFilesFromDir finds and loads any *.inc.js packages in the build context
445-
// directory.
446-
func jsFilesFromDir(bctx *build.Context, dir string) ([]JSFile, error) {
447-
files, err := buildutil.ReadDir(bctx, dir)
448-
if err != nil {
449-
return nil, err
450-
}
451-
var jsFiles []JSFile
452-
for _, file := range files {
453-
if !strings.HasSuffix(file.Name(), ".inc.js") || file.IsDir() {
454-
continue
455-
}
456-
if file.Name()[0] == '_' || file.Name()[0] == '.' {
457-
continue // Skip "hidden" files that are typically ignored by the Go build system.
458-
}
459-
460-
path := buildutil.JoinPath(bctx, dir, file.Name())
461-
f, err := buildutil.OpenFile(bctx, path)
462-
if err != nil {
463-
return nil, fmt.Errorf("failed to open %s from %v: %w", path, bctx, err)
464-
}
465-
defer f.Close()
466-
467-
content, err := io.ReadAll(f)
468-
if err != nil {
469-
return nil, fmt.Errorf("failed to read %s from %v: %w", path, bctx, err)
470-
}
471-
472-
jsFiles = append(jsFiles, JSFile{
473-
Path: path,
474-
ModTime: file.ModTime(),
475-
Content: content,
476-
})
477-
}
478-
return jsFiles, nil
479-
}

0 commit comments

Comments
 (0)