From 78745daf41c9ed67c548488a6f26b1acbd968165 Mon Sep 17 00:00:00 2001 From: Nevkontakte Date: Wed, 16 Feb 2022 22:00:05 +0000 Subject: [PATCH] Improved build cache. Previously GopherJS used to store its build cache under $GOPATH/pkg/$GOOS_$GOARCH, and after Go Modules introduction some parts of it were stored under os.UserCacheDir(). Starting from this commit, *all* build cache is located under os.UserCacheDir(), in a manner similar to the modern Go tool. The cache is keyed by a set of build options (such as minification, compiler version, etc.) to ensure that incompatible archives aren't picked up (see gopherjs#440 for example). This change doesn't solve *all* possible cache-related issues (for example, it still relies on timestamps to invalidate the cache, see gopherjs#805), but should eliminate a large class of confusing failure modes. --- build/build.go | 239 +++++++++++++++++--------------------- build/cache.go | 35 ------ build/cache/cache.go | 146 +++++++++++++++++++++++ build/cache/cache_test.go | 78 +++++++++++++ build/context.go | 28 ----- circle.yml | 2 +- compiler/compiler.go | 26 +++-- compiler/package.go | 2 + tool.go | 21 ++-- 9 files changed, 367 insertions(+), 210 deletions(-) delete mode 100644 build/cache.go create mode 100644 build/cache/cache.go create mode 100644 build/cache/cache_test.go diff --git a/build/build.go b/build/build.go index 1118ce233..a43f79d9b 100644 --- a/build/build.go +++ b/build/build.go @@ -31,6 +31,7 @@ import ( "github.com/shurcooL/httpfs/vfsutil" "golang.org/x/tools/go/buildutil" + "github.com/gopherjs/gopherjs/build/cache" _ "github.com/gopherjs/gopherjs/build/versionhack" // go/build release tags hack. ) @@ -108,29 +109,7 @@ func Import(path string, mode build.ImportMode, installSuffix string, buildTags wd = "" } xctx := NewBuildContext(installSuffix, buildTags) - return importWithSrcDir(xctx, path, wd, mode, installSuffix) -} - -func importWithSrcDir(xctx XContext, path string, srcDir string, mode build.ImportMode, installSuffix string) (*PackageData, error) { - pkg, err := xctx.Import(path, srcDir, mode) - if err != nil { - return nil, err - } - - if pkg.IsCommand() { - pkg.PkgObj = filepath.Join(pkg.BinDir, filepath.Base(pkg.ImportPath)+".js") - } - - if _, err := os.Stat(pkg.PkgObj); os.IsNotExist(err) && strings.HasPrefix(pkg.PkgObj, DefaultGOROOT) { - // fall back to GOPATH - firstGopathWorkspace := filepath.SplitList(build.Default.GOPATH)[0] // TODO: Need to check inside all GOPATH workspaces. - gopathPkgObj := filepath.Join(firstGopathWorkspace, pkg.PkgObj[len(DefaultGOROOT):]) - if _, err := os.Stat(gopathPkgObj); err == nil { - pkg.PkgObj = gopathPkgObj - } - } - - return pkg, nil + return xctx.Import(path, wd, mode) } // excludeExecutable excludes all executable implementation .go files. @@ -352,6 +331,7 @@ type Options struct { Minify bool Color bool BuildTags []string + TestedPackage string } // PrintError message to the terminal. @@ -374,11 +354,13 @@ func (o *Options) PrintSuccess(format string, a ...interface{}) { // GopherJS requires. type PackageData struct { *build.Package - JSFiles []string - IsTest bool // IsTest is true if the package is being built for running tests. + JSFiles []string + // IsTest is true if the package is being built for running tests. + IsTest bool SrcModTime time.Time UpToDate bool - IsVirtual bool // If true, the package does not have a corresponding physical directory on disk. + // If true, the package does not have a corresponding physical directory on disk. + IsVirtual bool bctx *build.Context // The original build context this package came from. } @@ -420,16 +402,38 @@ func (p *PackageData) XTestPackage() *PackageData { } } +// InstallPath returns the path where "gopherjs install" command should place the +// generated output. +func (p *PackageData) InstallPath() string { + if p.IsCommand() { + name := filepath.Base(p.ImportPath) + ".js" + // For executable packages, mimic go tool behavior if possible. + if gobin := os.Getenv("GOBIN"); gobin != "" { + return filepath.Join(gobin, name) + } else if gopath := os.Getenv("GOPATH"); gopath != "" { + return filepath.Join(gopath, "bin", name) + } else if home, err := os.UserHomeDir(); err == nil { + return filepath.Join(home, "go", "bin", name) + } + } + return p.PkgObj +} + // Session manages internal state GopherJS requires to perform a build. // // This is the main interface to GopherJS build system. Session lifetime is // roughly equivalent to a single GopherJS tool invocation. type Session struct { - options *Options - xctx XContext - Archives map[string]*compiler.Archive - Types map[string]*types.Package - Watcher *fsnotify.Watcher + options *Options + xctx XContext + buildCache cache.BuildCache + + // Binary archives produced during the current session and assumed to be + // up to date with input sources and dependencies. In the -w ("watch") mode + // must be cleared upon entering watching. + UpToDateArchives map[string]*compiler.Archive + Types map[string]*types.Package + Watcher *fsnotify.Watcher } // NewSession creates a new GopherJS build session. @@ -448,10 +452,19 @@ func NewSession(options *Options) (*Session, error) { } s := &Session{ - options: options, - Archives: make(map[string]*compiler.Archive), + options: options, + UpToDateArchives: make(map[string]*compiler.Archive), } s.xctx = NewBuildContext(s.InstallSuffix(), s.options.BuildTags) + s.buildCache = cache.BuildCache{ + GOOS: s.xctx.GOOS(), + GOARCH: "js", + GOROOT: options.GOROOT, + GOPATH: options.GOPATH, + BuildTags: options.BuildTags, + Minify: options.Minify, + TestedPackage: options.TestedPackage, + } s.Types = make(map[string]*types.Package) if options.Watch { if out, err := exec.Command("ulimit", "-n").Output(); err == nil { @@ -496,7 +509,10 @@ func (s *Session) BuildFiles(filenames []string, pkgObj string, packagePath stri ImportPath: "main", Dir: packagePath, }, - bctx: &goCtx(s.InstallSuffix(), s.options.BuildTags).bctx, + // This ephemeral package doesn't have a unique import path to be used as a + // build cache key, so we never cache it. + SrcModTime: time.Now().Add(time.Hour), + bctx: &goCtx(s.InstallSuffix(), s.options.BuildTags).bctx, } for _, file := range filenames { @@ -530,7 +546,7 @@ func (s *Session) BuildImportPath(path string) (*compiler.Archive, error) { // Relative import paths are interpreted relative to the passed srcDir. If // srcDir is empty, current working directory is assumed. func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*PackageData, *compiler.Archive, error) { - pkg, err := importWithSrcDir(s.xctx, path, srcDir, 0, s.InstallSuffix()) + pkg, err := s.xctx.Import(path, srcDir, 0) if s.Watcher != nil && pkg != nil { // add watch even on error s.Watcher.Add(pkg.Dir) } @@ -548,94 +564,70 @@ func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*Packag // BuildPackage compiles an already loaded package. func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { - if archive, ok := s.Archives[pkg.ImportPath]; ok { + if archive, ok := s.UpToDateArchives[pkg.ImportPath]; ok { return archive, nil } - if pkg.PkgObj != "" { - var fileInfo os.FileInfo - gopherjsBinary, err := os.Executable() - if err == nil { - fileInfo, err = os.Stat(gopherjsBinary) - if err == nil { - pkg.SrcModTime = fileInfo.ModTime() - } + var fileInfo os.FileInfo + gopherjsBinary, err := os.Executable() + if err == nil { + fileInfo, err = os.Stat(gopherjsBinary) + if err == nil && fileInfo.ModTime().After(pkg.SrcModTime) { + pkg.SrcModTime = fileInfo.ModTime() } + } + if err != nil { + os.Stderr.WriteString("Could not get GopherJS binary's modification timestamp. Please report issue.\n") + pkg.SrcModTime = time.Now() + } + + for _, importedPkgPath := range pkg.Imports { + if importedPkgPath == "unsafe" { + continue + } + importedPkg, _, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir) if err != nil { - os.Stderr.WriteString("Could not get GopherJS binary's modification timestamp. Please report issue.\n") - pkg.SrcModTime = time.Now() + return nil, err } - for _, importedPkgPath := range pkg.Imports { - if importedPkgPath == "unsafe" { - continue - } - importedPkg, _, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir) - if err != nil { - return nil, err - } - impModTime := importedPkg.SrcModTime - if impModTime.After(pkg.SrcModTime) { - pkg.SrcModTime = impModTime - } + impModTime := importedPkg.SrcModTime + if impModTime.After(pkg.SrcModTime) { + pkg.SrcModTime = impModTime } + } - for _, name := range append(pkg.GoFiles, pkg.JSFiles...) { - fileInfo, err := statFile(filepath.Join(pkg.Dir, name)) - if err != nil { - return nil, err - } - if fileInfo.ModTime().After(pkg.SrcModTime) { - pkg.SrcModTime = fileInfo.ModTime() - } + for _, name := range append(pkg.GoFiles, pkg.JSFiles...) { + fileInfo, err := statFile(filepath.Join(pkg.Dir, name)) + if err != nil { + return nil, err } + if fileInfo.ModTime().After(pkg.SrcModTime) { + pkg.SrcModTime = fileInfo.ModTime() + } + } - pkgObjFileInfo, err := os.Stat(pkg.PkgObj) - if err == nil && !pkg.SrcModTime.After(pkgObjFileInfo.ModTime()) { - // package object is up to date, load from disk if library - pkg.UpToDate = true - if pkg.IsCommand() { - return nil, nil - } - - objFile, err := os.Open(pkg.PkgObj) - if err != nil { - return nil, err - } - defer objFile.Close() - - archive, err := compiler.ReadArchive(pkg.PkgObj, pkg.ImportPath, objFile, s.Types) - if err != nil { - return nil, err - } - - s.Archives[pkg.ImportPath] = archive - return archive, err + archive := s.buildCache.LoadArchive(pkg.ImportPath) + if archive != nil && !pkg.SrcModTime.After(archive.BuildTime) { + if err := archive.RegisterTypes(s.Types); err != nil { + panic(fmt.Errorf("Failed to load type information from %v: %w", archive, err)) } + s.UpToDateArchives[pkg.ImportPath] = archive + // Existing archive is up to date, no need to build it from scratch. + return archive, nil } + // Existing archive is out of date or doesn't exist, let's build the package. fileSet := token.NewFileSet() files, err := parseAndAugment(s.xctx, pkg, pkg.IsTest, fileSet) if err != nil { return nil, err } - localImportPathCache := make(map[string]*compiler.Archive) importContext := &compiler.ImportContext{ Packages: s.Types, - Import: func(path string) (*compiler.Archive, error) { - if archive, ok := localImportPathCache[path]; ok { - return archive, nil - } - _, archive, err := s.buildImportPathWithSrcDir(path, pkg.Dir) - if err != nil { - return nil, err - } - localImportPathCache[path] = archive - return archive, nil - }, + Import: s.ImportResolverFor(pkg), } - archive, err := compiler.Compile(pkg.ImportPath, files, fileSet, importContext, s.options.Minify) + archive, err = compiler.Compile(pkg.ImportPath, files, fileSet, importContext, s.options.Minify) if err != nil { return nil, err } @@ -654,40 +646,22 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { fmt.Println(pkg.ImportPath) } - s.Archives[pkg.ImportPath] = archive - - if pkg.PkgObj == "" || pkg.IsCommand() { - return archive, nil - } - - if err := s.writeLibraryPackage(archive, pkg.PkgObj); err != nil { - if strings.HasPrefix(pkg.PkgObj, s.options.GOROOT) { - // fall back to first GOPATH workspace - firstGopathWorkspace := filepath.SplitList(s.options.GOPATH)[0] - if err := s.writeLibraryPackage(archive, filepath.Join(firstGopathWorkspace, pkg.PkgObj[len(s.options.GOROOT):])); err != nil { - return nil, err - } - return archive, nil - } - return nil, err - } + s.buildCache.StoreArchive(archive) + s.UpToDateArchives[pkg.ImportPath] = archive return archive, nil } -// writeLibraryPackage writes a compiled package archive to disk at pkgObj path. -func (s *Session) writeLibraryPackage(archive *compiler.Archive, pkgObj string) error { - if err := os.MkdirAll(filepath.Dir(pkgObj), 0777); err != nil { - return err - } - - objFile, err := os.Create(pkgObj) - if err != nil { - return err +// ImportResolverFor returns a function which returns a compiled package archive +// given an import path. +func (s *Session) ImportResolverFor(pkg *PackageData) func(string) (*compiler.Archive, error) { + return func(path string) (*compiler.Archive, error) { + if archive, ok := s.UpToDateArchives[path]; ok { + return archive, nil + } + _, archive, err := s.buildImportPathWithSrcDir(path, pkg.Dir) + return archive, err } - defer objFile.Close() - - return compiler.WriteArchive(archive, objFile) } // WriteCommandPackage writes the final JavaScript output file at pkgObj path. @@ -719,7 +693,7 @@ func (s *Session) WriteCommandPackage(archive *compiler.Archive, pkgObj string) } deps, err := compiler.ImportDependencies(archive, func(path string) (*compiler.Archive, error) { - if archive, ok := s.Archives[path]; ok { + if archive, ok := s.UpToDateArchives[path]; ok { return archive, nil } _, archive, err := s.buildImportPathWithSrcDir(path, "") @@ -788,6 +762,11 @@ func hasGopathPrefix(file, gopath string) (hasGopathPrefix bool, prefixLen int) // WaitForChange watches file system events and returns if either when one of // the source files is modified. func (s *Session) WaitForChange() { + // Will need to re-validate up-to-dateness of all archives, so flush them from + // memory. + s.UpToDateArchives = map[string]*compiler.Archive{} + s.Types = map[string]*types.Package{} + s.options.PrintSuccess("watching for changes...\n") for { select { diff --git a/build/cache.go b/build/cache.go deleted file mode 100644 index e121fbdbd..000000000 --- a/build/cache.go +++ /dev/null @@ -1,35 +0,0 @@ -package build - -import ( - "crypto/sha256" - "fmt" - "go/build" - "os" - "path/filepath" -) - -// cachePath is the base path for GopherJS's own build cache. -// -// It serves a similar function to the Go build cache, but is a lot more -// simplistic and therefore not compatible with Go. We use this cache directory -// to store build artifacts for packages loaded from a module, for which PkgObj -// provided by go/build points inside the module source tree, which can cause -// inconvenience with version control, etc. -var cachePath = func() string { - path, err := os.UserCacheDir() - if err == nil { - return filepath.Join(path, "gopherjs", "build_cache") - } - - return filepath.Join(build.Default.GOPATH, "pkg", "gopherjs_build_cache") -}() - -// cachedPath returns a location inside the build cache for a given PkgObj path -// returned by go/build. -func cachedPath(orig string) string { - if orig == "" { - panic("CachedPath() must not be used with an empty string") - } - sum := fmt.Sprintf("%x", sha256.Sum256([]byte(orig))) - return filepath.Join(cachePath, sum[0:2], sum) -} diff --git a/build/cache/cache.go b/build/cache/cache.go new file mode 100644 index 000000000..728962eb0 --- /dev/null +++ b/build/cache/cache.go @@ -0,0 +1,146 @@ +// Package cache solves one of the hardest computer science problems in +// application to GopherJS compiler outputs. +package cache + +import ( + "crypto/sha256" + "fmt" + "go/build" + "os" + "path" + "path/filepath" + + "github.com/gopherjs/gopherjs/compiler" +) + +// cacheRoot is the base path for GopherJS's own build cache. +// +// It serves a similar function to the Go build cache, but is a lot more +// simplistic and therefore not compatible with Go. We use this cache directory +// to store build artifacts for packages loaded from a module, for which PkgObj +// provided by go/build points inside the module source tree, which can cause +// inconvenience with version control, etc. +var cacheRoot = func() string { + path, err := os.UserCacheDir() + if err == nil { + return filepath.Join(path, "gopherjs", "build_cache") + } + + return filepath.Join(build.Default.GOPATH, "pkg", "gopherjs_build_cache") +}() + +// cachedPath returns a location inside the build cache for a given set of key +// strings. The set of keys must uniquely identify cacheable object. Prefer +// using more specific functions to ensure key consistency. +func cachedPath(keys ...string) string { + key := path.Join(keys...) + if key == "" { + panic("CachedPath() must not be used with an empty string") + } + sum := fmt.Sprintf("%x", sha256.Sum256([]byte(key))) + return filepath.Join(cacheRoot, sum[0:2], sum) +} + +// Clear the cache. This will remove *all* cached artifacts from *all* build +// configurations. +func Clear() error { + return os.RemoveAll(cacheRoot) +} + +// BuildCache manages build artifacts that are cached for incremental builds. +// +// Cache is designed to be non-durable: any store and load errors are swallowed +// and simply lead to a cache miss. The caller must be able to handle cache +// misses. Nil pointer to BuildCache is valid and simply disables caching. +// +// BuildCache struct fields represent build parameters which change invalidates +// the cache. For example, any artifacts that were cached for a minified build +// must not be reused for a non-minified build. GopherJS version change also +// invalidates the cache. It is callers responsibility to ensure that artifacts +// passed the the StoreArchive function were generated with the same build +// parameters as the cache is configured. +// +// There is no upper limit for the total cache size. It can be cleared +// programmatically via the Clear() function, or the user can just delete the +// directory if it grows too big. +// +// TODO(nevkontakte): changes in the input sources or dependencies doesn't +// currently invalidate the cache. This is handled at the higher level by +// checking cached archive timestamp against loaded package modification time. +// +// TODO(nevkontakte): this cache could benefit from checksum integrity checks. +type BuildCache struct { + GOOS string + GOARCH string + GOROOT string + GOPATH string + BuildTags []string + Minify bool + // When building for tests, import path of the package being tested. The + // package under test is built with *_test.go sources included, and since it + // may be imported by other packages in the binary we can't reuse the "normal" + // cache. + TestedPackage string +} + +func (bc BuildCache) String() string { + return fmt.Sprintf("%#v", bc) +} + +// StoreArchive compiled archive in the cache. Any error inside this method +// will cause the cache not to be persisted. +func (bc *BuildCache) StoreArchive(a *compiler.Archive) { + if bc == nil { + return // Caching is disabled. + } + path := cachedPath(bc.archiveKey(a.ImportPath)) + if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil { + return + } + // Write the archive in a temporary file first to avoid concurrency errors. + f, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path)) + if err != nil { + return + } + defer f.Close() + if err := compiler.WriteArchive(a, f); err != nil { + // Make sure we don't leave a half-written archive behind. + os.Remove(f.Name()) + } + f.Close() + // Rename fully written file into its permanent name. + os.Rename(f.Name(), path) +} + +// LoadArchive returns a previously cached archive of the given package or nil +// if it wasn't previously stored. +// +// The returned archive would have been built with the same configuration as +// the build cache was. +func (bc *BuildCache) LoadArchive(importPath string) *compiler.Archive { + if bc == nil { + return nil // Caching is disabled. + } + path := cachedPath(bc.archiveKey(importPath)) + f, err := os.Open(path) + if err != nil { + return nil // Cache miss. + } + defer f.Close() + a, err := compiler.ReadArchive(importPath, f) + if err != nil { + return nil // Invalid/corrupted archive, cache miss. + } + return a +} + +// commonKey returns a part of the cache key common for all artifacts generated +// under a given BuildCache configuration. +func (bc *BuildCache) commonKey() string { + return fmt.Sprintf("%#v + %v", *bc, compiler.Version) +} + +// archiveKey returns a full cache key for a package's compiled archive. +func (bc *BuildCache) archiveKey(importPath string) string { + return path.Join("archive", bc.commonKey(), importPath) +} diff --git a/build/cache/cache_test.go b/build/cache/cache_test.go new file mode 100644 index 000000000..fd89ec187 --- /dev/null +++ b/build/cache/cache_test.go @@ -0,0 +1,78 @@ +package cache + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/gopherjs/gopherjs/compiler" +) + +func TestStore(t *testing.T) { + cacheForTest(t) + + want := &compiler.Archive{ + ImportPath: "fake/package", + Imports: []string{"fake/dep"}, + } + + bc := BuildCache{} + if got := bc.LoadArchive(want.ImportPath); got != nil { + t.Errorf("Got: %s was found in the cache. Want: empty cache.", got.ImportPath) + } + bc.StoreArchive(want) + got := bc.LoadArchive(want.ImportPath) + if got == nil { + t.Errorf("Got: %s wan not found in the cache. Want: archive is can be loaded after store.", want.ImportPath) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("Loaded archive is different from stored (-want,+got):\n%s", diff) + } + + // Make sure the package names are a part of the cache key. + if got := bc.LoadArchive("fake/other"); got != nil { + t.Errorf("Got: fake/other was found in cache: %#v. Want: nil for packages that weren't cached.", got) + } +} + +func TestInvalidation(t *testing.T) { + cacheForTest(t) + + tests := []struct { + cache1 BuildCache + cache2 BuildCache + }{ + { + cache1: BuildCache{Minify: true}, + cache2: BuildCache{Minify: false}, + }, { + cache1: BuildCache{GOOS: "dos"}, + cache2: BuildCache{GOOS: "amiga"}, + }, { + cache1: BuildCache{GOARCH: "m68k"}, + cache2: BuildCache{GOARCH: "mos6502"}, + }, { + cache1: BuildCache{GOROOT: "here"}, + cache2: BuildCache{GOROOT: "there"}, + }, { + cache1: BuildCache{GOPATH: "home"}, + cache2: BuildCache{GOPATH: "away"}, + }, + } + + for _, test := range tests { + a := &compiler.Archive{ImportPath: "package/fake"} + test.cache1.StoreArchive(a) + + if got := test.cache2.LoadArchive(a.ImportPath); got != nil { + t.Logf("-cache1,+cache2:\n%s", cmp.Diff(test.cache1, test.cache2)) + t.Errorf("Got: %v loaded from cache. Want: build parameter change invalidates cache.", got) + } + } +} + +func cacheForTest(t *testing.T) { + t.Helper() + originalRoot := cacheRoot + t.Cleanup(func() { cacheRoot = originalRoot }) + cacheRoot = t.TempDir() +} diff --git a/build/context.go b/build/context.go index b980aa7de..54c1a4db5 100644 --- a/build/context.go +++ b/build/context.go @@ -51,7 +51,6 @@ func (sc simpleCtx) Import(importPath string, srcDir string, mode build.ImportMo if err != nil { return nil, fmt.Errorf("failed to enumerate .inc.js files in %s: %w", pkg.Dir, err) } - pkg.PkgObj = sc.rewritePkgObj(pkg.PkgObj) if !path.IsAbs(pkg.Dir) { pkg.Dir = mustAbs(pkg.Dir) } @@ -252,33 +251,6 @@ func (sc simpleCtx) applyPostloadTweaks(pkg *build.Package) *build.Package { return pkg } -func (sc simpleCtx) rewritePkgObj(orig string) string { - if orig == "" { - return orig - } - - goroot := mustAbs(sc.bctx.GOROOT) - gopath := mustAbs(sc.bctx.GOPATH) - orig = mustAbs(orig) - - if strings.HasPrefix(orig, filepath.Join(gopath, "pkg", "mod")) { - // Go toolchain makes sources under GOPATH/pkg/mod readonly, so we can't - // store our artifacts there. - return cachedPath(orig) - } - - allowed := []string{goroot, gopath} - for _, prefix := range allowed { - if strings.HasPrefix(orig, prefix) { - // Traditional GOPATH-style locations for build artifacts are ok to use. - return orig - } - } - - // Everything else also goes into the cache just in case. - return cachedPath(orig) -} - var defaultBuildTags = []string{ "netgo", // See https://godoc.org/net#hdr-Name_Resolution. "purego", // See https://golang.org/issues/23172. diff --git a/circle.yml b/circle.yml index f129360cf..b9536cb0c 100644 --- a/circle.yml +++ b/circle.yml @@ -84,7 +84,7 @@ jobs: - run: name: Smoke tests command: | - gopherjs install -v net/http # Should build successfully. + gopherjs build -v net/http # Should build successfully. gopherjs test -v fmt log # Should catch problems with test execution and source maps. - run: name: go test ... diff --git a/compiler/compiler.go b/compiler/compiler.go index ec38ed197..22edb9e3b 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -15,6 +15,7 @@ import ( "go/types" "io" "strings" + "time" "github.com/gopherjs/gopherjs/compiler/prelude" "golang.org/x/tools/go/gcexportdata" @@ -80,6 +81,21 @@ type Archive struct { Minified bool // A list of go:linkname directives encountered in the package. GoLinknames []GoLinkname + // Time when this archive was built. + BuildTime time.Time +} + +func (a Archive) String() string { + return fmt.Sprintf("compiler.Archive{%s}", a.ImportPath) +} + +// RegisterTypes adds package type information from the archive into the provided map. +func (a *Archive) RegisterTypes(packages map[string]*types.Package) error { + var err error + // TODO(nevkontakte): Should this be shared throughout the build? + fset := token.NewFileSet() + packages[a.ImportPath], err = gcexportdata.Read(bytes.NewReader(a.ExportData), fset, packages, a.ImportPath) + return err } // Decl represents a package-level symbol (e.g. a function, variable or type). @@ -359,21 +375,17 @@ func WritePkgCode(pkg *Archive, dceSelection map[*Decl]struct{}, gls goLinknameS return nil } -func ReadArchive(filename, path string, r io.Reader, packages map[string]*types.Package) (*Archive, error) { +// ReadArchive reads serialized compiled archive of the importPath package. +func ReadArchive(path string, r io.Reader) (*Archive, error) { var a Archive if err := gob.NewDecoder(r).Decode(&a); err != nil { return nil, err } - var err error - packages[path], err = gcexportdata.Read(bytes.NewReader(a.ExportData), token.NewFileSet(), packages, path) - if err != nil { - return nil, err - } - return &a, nil } +// WriteArchive writes compiled package archive on disk for later reuse. func WriteArchive(a *Archive, w io.Writer) error { return gob.NewEncoder(w).Encode(a) } diff --git a/compiler/package.go b/compiler/package.go index c68957450..8bc2a160e 100644 --- a/compiler/package.go +++ b/compiler/package.go @@ -10,6 +10,7 @@ import ( "go/types" "sort" "strings" + "time" "github.com/gopherjs/gopherjs/compiler/analysis" "github.com/gopherjs/gopherjs/compiler/astutil" @@ -595,6 +596,7 @@ func Compile(importPath string, files []*ast.File, fileSet *token.FileSet, impor FileSet: encodedFileSet.Bytes(), Minified: minify, GoLinknames: goLinknames, + BuildTime: time.Now(), }, nil } diff --git a/tool.go b/tool.go index 84537a26f..dcae56119 100644 --- a/tool.go +++ b/tool.go @@ -224,7 +224,7 @@ func main() { } if pkg.IsCommand() && !pkg.UpToDate { - if err := s.WriteCommandPackage(archive, pkg.PkgObj); err != nil { + if err := s.WriteCommandPackage(archive, pkg.InstallPath()); err != nil { return err } } @@ -373,7 +373,9 @@ func main() { fmt.Printf("? \t%s\t[no test files]\n", pkg.ImportPath) continue } - s, err := gbuild.NewSession(options) + localOpts := options + localOpts.TestedPackage = pkg.ImportPath + s, err := gbuild.NewSession(localOpts) if err != nil { return err } @@ -416,16 +418,17 @@ func main() { return err } + mainPkg := &gbuild.PackageData{ + Package: &build.Package{ + ImportPath: pkg.ImportPath + ".testmain", + Name: "main", + }, + } importContext := &compiler.ImportContext{ Packages: s.Types, - Import: func(path string) (*compiler.Archive, error) { - if path == pkg.ImportPath || path == pkg.ImportPath+"_test" { - return s.Archives[path], nil - } - return s.BuildImportPath(path) - }, + Import: s.ImportResolverFor(mainPkg), } - mainPkgArchive, err := compiler.Compile("main", []*ast.File{mainFile}, fset, importContext, options.Minify) + mainPkgArchive, err := compiler.Compile(mainPkg.ImportPath, []*ast.File{mainFile}, fset, importContext, options.Minify) if err != nil { return err }