Skip to content

Commit dad6ef5

Browse files
nevkontaktenevkontakte
committed
Improved build cache.
Previously GopherJS used to store its build cache under $GOPATH/pkg/$GOOS_$GOARCH, and after Go Modules introduction some parts of it were stored under os.UserCacheDir(). Starting from this commit, *all* build cache is located under os.UserCacheDir(), in a manner similar to the modern Go tool. The cache is keyed by a set of build options (such as minification, compiler version, etc.) to ensure that incompatible archives aren't picked up (see #440 for example). This change doesn't solve *all* possible cache-related issues (for example, it still relies on timestamps to invalidate the cache, see #805), but should eliminate a large class of confusing failure modes.
1 parent 15ed2e8 commit dad6ef5

File tree

9 files changed

+360
-210
lines changed

9 files changed

+360
-210
lines changed

‎build/build.go

Lines changed: 106 additions & 130 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
"github.com/shurcooL/httpfs/vfsutil"
3232
"golang.org/x/tools/go/buildutil"
3333

34+
"github.com/gopherjs/gopherjs/build/cache"
3435
_ "github.com/gopherjs/gopherjs/build/versionhack" // go/build release tags hack.
3536
)
3637

@@ -108,29 +109,7 @@ func Import(path string, mode build.ImportMode, installSuffix string, buildTags
108109
wd = ""
109110
}
110111
xctx := NewBuildContext(installSuffix, buildTags)
111-
return importWithSrcDir(xctx, path, wd, mode, installSuffix)
112-
}
113-
114-
func importWithSrcDir(xctx XContext, path string, srcDir string, mode build.ImportMode, installSuffix string) (*PackageData, error) {
115-
pkg, err := xctx.Import(path, srcDir, mode)
116-
if err != nil {
117-
return nil, err
118-
}
119-
120-
if pkg.IsCommand() {
121-
pkg.PkgObj = filepath.Join(pkg.BinDir, filepath.Base(pkg.ImportPath)+".js")
122-
}
123-
124-
if _, err := os.Stat(pkg.PkgObj); os.IsNotExist(err) && strings.HasPrefix(pkg.PkgObj, DefaultGOROOT) {
125-
// fall back to GOPATH
126-
firstGopathWorkspace := filepath.SplitList(build.Default.GOPATH)[0] // TODO: Need to check inside all GOPATH workspaces.
127-
gopathPkgObj := filepath.Join(firstGopathWorkspace, pkg.PkgObj[len(DefaultGOROOT):])
128-
if _, err := os.Stat(gopathPkgObj); err == nil {
129-
pkg.PkgObj = gopathPkgObj
130-
}
131-
}
132-
133-
return pkg, nil
112+
return xctx.Import(path, wd, mode)
134113
}
135114

136115
// excludeExecutable excludes all executable implementation .go files.
@@ -352,6 +331,7 @@ type Options struct {
352331
Minify bool
353332
Color bool
354333
BuildTags []string
334+
TestedPackage string
355335
}
356336

357337
// PrintError message to the terminal.
@@ -374,11 +354,13 @@ func (o *Options) PrintSuccess(format string, a ...interface{}) {
374354
// GopherJS requires.
375355
type PackageData struct {
376356
*build.Package
377-
JSFiles []string
378-
IsTest bool // IsTest is true if the package is being built for running tests.
357+
JSFiles []string
358+
// IsTest is true if the package is being built for running tests.
359+
IsTest bool
379360
SrcModTime time.Time
380361
UpToDate bool
381-
IsVirtual bool // If true, the package does not have a corresponding physical directory on disk.
362+
// If true, the package does not have a corresponding physical directory on disk.
363+
IsVirtual bool
382364

383365
bctx *build.Context // The original build context this package came from.
384366
}
@@ -420,16 +402,38 @@ func (p *PackageData) XTestPackage() *PackageData {
420402
}
421403
}
422404

405+
// InstallPath returns the path where "gopherjs install" command should place the
406+
// generated output.
407+
func (p *PackageData) InstallPath() string {
408+
if p.IsCommand() {
409+
name := filepath.Base(p.ImportPath) + ".js"
410+
// For executable packages, mimic go tool behavior if possible.
411+
if gobin := os.Getenv("GOBIN"); gobin != "" {
412+
return filepath.Join(gobin, name)
413+
} else if gopath := os.Getenv("GOPATH"); gopath != "" {
414+
return filepath.Join(gopath, "bin", name)
415+
} else if home, err := os.UserHomeDir(); err == nil {
416+
return filepath.Join(home, "go", "bin", name)
417+
}
418+
}
419+
return p.PkgObj
420+
}
421+
423422
// Session manages internal state GopherJS requires to perform a build.
424423
//
425424
// This is the main interface to GopherJS build system. Session lifetime is
426425
// roughly equivalent to a single GopherJS tool invocation.
427426
type Session struct {
428-
options *Options
429-
xctx XContext
430-
Archives map[string]*compiler.Archive
431-
Types map[string]*types.Package
432-
Watcher *fsnotify.Watcher
427+
options *Options
428+
xctx XContext
429+
buildCache cache.BuildCache
430+
431+
// Binary archives produced during the current session and assumed to be
432+
// up to date with input sources and dependencies. In the -w ("watch") mode
433+
// must be cleared upon entering watching.
434+
UpToDateArchives map[string]*compiler.Archive
435+
Types map[string]*types.Package
436+
Watcher *fsnotify.Watcher
433437
}
434438

435439
// NewSession creates a new GopherJS build session.
@@ -448,10 +452,19 @@ func NewSession(options *Options) (*Session, error) {
448452
}
449453

450454
s := &Session{
451-
options: options,
452-
Archives: make(map[string]*compiler.Archive),
455+
options: options,
456+
UpToDateArchives: make(map[string]*compiler.Archive),
453457
}
454458
s.xctx = NewBuildContext(s.InstallSuffix(), s.options.BuildTags)
459+
s.buildCache = cache.BuildCache{
460+
GOOS: s.xctx.GOOS(),
461+
GOARCH: "js",
462+
GOROOT: options.GOROOT,
463+
GOPATH: options.GOPATH,
464+
BuildTags: options.BuildTags,
465+
Minify: options.Minify,
466+
TestedPackage: options.TestedPackage,
467+
}
455468
s.Types = make(map[string]*types.Package)
456469
if options.Watch {
457470
if out, err := exec.Command("ulimit", "-n").Output(); err == nil {
@@ -496,7 +509,10 @@ func (s *Session) BuildFiles(filenames []string, pkgObj string, packagePath stri
496509
ImportPath: "main",
497510
Dir: packagePath,
498511
},
499-
bctx: &goCtx(s.InstallSuffix(), s.options.BuildTags).bctx,
512+
// This ephemeral package doesn't have a unique import path to be used as a
513+
// build cache key, so we never cache it.
514+
SrcModTime: time.Now().Add(time.Hour),
515+
bctx: &goCtx(s.InstallSuffix(), s.options.BuildTags).bctx,
500516
}
501517

502518
for _, file := range filenames {
@@ -530,7 +546,7 @@ func (s *Session) BuildImportPath(path string) (*compiler.Archive, error) {
530546
// Relative import paths are interpreted relative to the passed srcDir. If
531547
// srcDir is empty, current working directory is assumed.
532548
func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*PackageData, *compiler.Archive, error) {
533-
pkg, err := importWithSrcDir(s.xctx, path, srcDir, 0, s.InstallSuffix())
549+
pkg, err := s.xctx.Import(path, srcDir, 0)
534550
if s.Watcher != nil && pkg != nil { // add watch even on error
535551
s.Watcher.Add(pkg.Dir)
536552
}
@@ -548,94 +564,70 @@ func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*Packag
548564

549565
// BuildPackage compiles an already loaded package.
550566
func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
551-
if archive, ok := s.Archives[pkg.ImportPath]; ok {
567+
if archive, ok := s.UpToDateArchives[pkg.ImportPath]; ok {
552568
return archive, nil
553569
}
554570

555-
if pkg.PkgObj != "" {
556-
var fileInfo os.FileInfo
557-
gopherjsBinary, err := os.Executable()
558-
if err == nil {
559-
fileInfo, err = os.Stat(gopherjsBinary)
560-
if err == nil {
561-
pkg.SrcModTime = fileInfo.ModTime()
562-
}
571+
var fileInfo os.FileInfo
572+
gopherjsBinary, err := os.Executable()
573+
if err == nil {
574+
fileInfo, err = os.Stat(gopherjsBinary)
575+
if err == nil && fileInfo.ModTime().After(pkg.SrcModTime) {
576+
pkg.SrcModTime = fileInfo.ModTime()
563577
}
578+
}
579+
if err != nil {
580+
os.Stderr.WriteString("Could not get GopherJS binary's modification timestamp. Please report issue.\n")
581+
pkg.SrcModTime = time.Now()
582+
}
583+
584+
for _, importedPkgPath := range pkg.Imports {
585+
if importedPkgPath == "unsafe" {
586+
continue
587+
}
588+
importedPkg, _, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir)
564589
if err != nil {
565-
os.Stderr.WriteString("Could not get GopherJS binary's modification timestamp. Please report issue.\n")
566-
pkg.SrcModTime = time.Now()
590+
return nil, err
567591
}
568592

569-
for _, importedPkgPath := range pkg.Imports {
570-
if importedPkgPath == "unsafe" {
571-
continue
572-
}
573-
importedPkg, _, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir)
574-
if err != nil {
575-
return nil, err
576-
}
577-
impModTime := importedPkg.SrcModTime
578-
if impModTime.After(pkg.SrcModTime) {
579-
pkg.SrcModTime = impModTime
580-
}
593+
impModTime := importedPkg.SrcModTime
594+
if impModTime.After(pkg.SrcModTime) {
595+
pkg.SrcModTime = impModTime
581596
}
597+
}
582598

583-
for _, name := range append(pkg.GoFiles, pkg.JSFiles...) {
584-
fileInfo, err := statFile(filepath.Join(pkg.Dir, name))
585-
if err != nil {
586-
return nil, err
587-
}
588-
if fileInfo.ModTime().After(pkg.SrcModTime) {
589-
pkg.SrcModTime = fileInfo.ModTime()
590-
}
599+
for _, name := range append(pkg.GoFiles, pkg.JSFiles...) {
600+
fileInfo, err := statFile(filepath.Join(pkg.Dir, name))
601+
if err != nil {
602+
return nil, err
591603
}
604+
if fileInfo.ModTime().After(pkg.SrcModTime) {
605+
pkg.SrcModTime = fileInfo.ModTime()
606+
}
607+
}
592608

593-
pkgObjFileInfo, err := os.Stat(pkg.PkgObj)
594-
if err == nil && !pkg.SrcModTime.After(pkgObjFileInfo.ModTime()) {
595-
// package object is up to date, load from disk if library
596-
pkg.UpToDate = true
597-
if pkg.IsCommand() {
598-
return nil, nil
599-
}
600-
601-
objFile, err := os.Open(pkg.PkgObj)
602-
if err != nil {
603-
return nil, err
604-
}
605-
defer objFile.Close()
606-
607-
archive, err := compiler.ReadArchive(pkg.PkgObj, pkg.ImportPath, objFile, s.Types)
608-
if err != nil {
609-
return nil, err
610-
}
611-
612-
s.Archives[pkg.ImportPath] = archive
613-
return archive, err
609+
archive := s.buildCache.LoadArchive(pkg.ImportPath)
610+
if archive != nil && !pkg.SrcModTime.After(archive.BuildTime) {
611+
if err := archive.RegisterTypes(s.Types); err != nil {
612+
panic(fmt.Errorf("Failed to load type information from %v: %w", archive, err))
614613
}
614+
s.UpToDateArchives[pkg.ImportPath] = archive
615+
// Existing archive is up to date, no need to build it from scratch.
616+
return archive, nil
615617
}
616618

619+
// Existing archive is out of date or doesn't exist, let's build the package.
617620
fileSet := token.NewFileSet()
618621
files, err := parseAndAugment(s.xctx, pkg, pkg.IsTest, fileSet)
619622
if err != nil {
620623
return nil, err
621624
}
622625

623-
localImportPathCache := make(map[string]*compiler.Archive)
624626
importContext := &compiler.ImportContext{
625627
Packages: s.Types,
626-
Import: func(path string) (*compiler.Archive, error) {
627-
if archive, ok := localImportPathCache[path]; ok {
628-
return archive, nil
629-
}
630-
_, archive, err := s.buildImportPathWithSrcDir(path, pkg.Dir)
631-
if err != nil {
632-
return nil, err
633-
}
634-
localImportPathCache[path] = archive
635-
return archive, nil
636-
},
628+
Import: s.ImportResolverFor(pkg),
637629
}
638-
archive, err := compiler.Compile(pkg.ImportPath, files, fileSet, importContext, s.options.Minify)
630+
archive, err = compiler.Compile(pkg.ImportPath, files, fileSet, importContext, s.options.Minify)
639631
if err != nil {
640632
return nil, err
641633
}
@@ -654,40 +646,19 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
654646
fmt.Println(pkg.ImportPath)
655647
}
656648

657-
s.Archives[pkg.ImportPath] = archive
658-
659-
if pkg.PkgObj == "" || pkg.IsCommand() {
660-
return archive, nil
661-
}
662-
663-
if err := s.writeLibraryPackage(archive, pkg.PkgObj); err != nil {
664-
if strings.HasPrefix(pkg.PkgObj, s.options.GOROOT) {
665-
// fall back to first GOPATH workspace
666-
firstGopathWorkspace := filepath.SplitList(s.options.GOPATH)[0]
667-
if err := s.writeLibraryPackage(archive, filepath.Join(firstGopathWorkspace, pkg.PkgObj[len(s.options.GOROOT):])); err != nil {
668-
return nil, err
669-
}
670-
return archive, nil
671-
}
672-
return nil, err
673-
}
649+
s.buildCache.StoreArchive(archive)
650+
s.UpToDateArchives[pkg.ImportPath] = archive
674651

675652
return archive, nil
676653
}
677654

678-
// writeLibraryPackage writes a compiled package archive to disk at pkgObj path.
679-
func (s *Session) writeLibraryPackage(archive *compiler.Archive, pkgObj string) error {
680-
if err := os.MkdirAll(filepath.Dir(pkgObj), 0777); err != nil {
681-
return err
682-
}
683-
684-
objFile, err := os.Create(pkgObj)
685-
if err != nil {
686-
return err
655+
// ImportResolverFor returns a function which returns a compiled package archive
656+
// given an import path.
657+
func (s *Session) ImportResolverFor(pkg *PackageData) func(string) (*compiler.Archive, error) {
658+
return func(path string) (*compiler.Archive, error) {
659+
_, archive, err := s.buildImportPathWithSrcDir(path, pkg.Dir)
660+
return archive, err
687661
}
688-
defer objFile.Close()
689-
690-
return compiler.WriteArchive(archive, objFile)
691662
}
692663

693664
// WriteCommandPackage writes the final JavaScript output file at pkgObj path.
@@ -719,7 +690,7 @@ func (s *Session) WriteCommandPackage(archive *compiler.Archive, pkgObj string)
719690
}
720691

721692
deps, err := compiler.ImportDependencies(archive, func(path string) (*compiler.Archive, error) {
722-
if archive, ok := s.Archives[path]; ok {
693+
if archive, ok := s.UpToDateArchives[path]; ok {
723694
return archive, nil
724695
}
725696
_, archive, err := s.buildImportPathWithSrcDir(path, "")
@@ -788,6 +759,11 @@ func hasGopathPrefix(file, gopath string) (hasGopathPrefix bool, prefixLen int)
788759
// WaitForChange watches file system events and returns if either when one of
789760
// the source files is modified.
790761
func (s *Session) WaitForChange() {
762+
// Will need to re-validate up-to-dateness of all archives, so flush them from
763+
// memory.
764+
s.UpToDateArchives = map[string]*compiler.Archive{}
765+
s.Types = map[string]*types.Package{}
766+
791767
s.options.PrintSuccess("watching for changes...\n")
792768
for {
793769
select {

‎build/cache.go

Lines changed: 0 additions & 35 deletions
This file was deleted.

0 commit comments

Comments
 (0)