Skip to content

Quality of life improvements for build cache #1107

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Feb 21, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 11 additions & 8 deletions build/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,7 @@ type Options struct {
Color bool
BuildTags []string
TestedPackage string
NoCache bool
}

// PrintError message to the terminal.
Expand Down Expand Up @@ -606,14 +607,16 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
}
}

archive := s.buildCache.LoadArchive(pkg.ImportPath)
if archive != nil && !pkg.SrcModTime.After(archive.BuildTime) {
if err := archive.RegisterTypes(s.Types); err != nil {
panic(fmt.Errorf("Failed to load type information from %v: %w", archive, err))
if !s.options.NoCache {
archive := s.buildCache.LoadArchive(pkg.ImportPath)
if archive != nil && !pkg.SrcModTime.After(archive.BuildTime) {
if err := archive.RegisterTypes(s.Types); err != nil {
panic(fmt.Errorf("Failed to load type information from %v: %w", archive, err))
}
s.UpToDateArchives[pkg.ImportPath] = archive
// Existing archive is up to date, no need to build it from scratch.
return archive, nil
}
s.UpToDateArchives[pkg.ImportPath] = archive
// Existing archive is up to date, no need to build it from scratch.
return archive, nil
}

// Existing archive is out of date or doesn't exist, let's build the package.
Expand All @@ -627,7 +630,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
Packages: s.Types,
Import: s.ImportResolverFor(pkg),
}
archive, err = compiler.Compile(pkg.ImportPath, files, fileSet, importContext, s.options.Minify)
archive, err := compiler.Compile(pkg.ImportPath, files, fileSet, importContext, s.options.Minify)
if err != nil {
return nil, err
}
Expand Down
17 changes: 16 additions & 1 deletion build/cache/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"path/filepath"

"github.com/gopherjs/gopherjs/compiler"
log "github.com/sirupsen/logrus"
)

// cacheRoot is the base path for GopherJS's own build cache.
Expand Down Expand Up @@ -95,21 +96,28 @@ func (bc *BuildCache) StoreArchive(a *compiler.Archive) {
}
path := cachedPath(bc.archiveKey(a.ImportPath))
if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil {
log.Warningf("Failed to create build cache directory: %v", err)
return
}
// Write the archive in a temporary file first to avoid concurrency errors.
f, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path))
if err != nil {
log.Warningf("Failed to temporary build cache file: %v", err)
return
}
defer f.Close()
if err := compiler.WriteArchive(a, f); err != nil {
log.Warningf("Failed to write build cache archive %q: %v", a, err)
// Make sure we don't leave a half-written archive behind.
os.Remove(f.Name())
return
}
f.Close()
// Rename fully written file into its permanent name.
os.Rename(f.Name(), path)
if err := os.Rename(f.Name(), path); err != nil {
log.Warningf("Failed to rename build cache archive to %q: %v", path, err)
}
log.Infof("Successfully stored build archive %q as %q.", a, path)
}

// LoadArchive returns a previously cached archive of the given package or nil
Expand All @@ -124,13 +132,20 @@ func (bc *BuildCache) LoadArchive(importPath string) *compiler.Archive {
path := cachedPath(bc.archiveKey(importPath))
f, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
log.Infof("No cached package archive for %q.", importPath)
} else {
log.Warningf("Failed to open cached package archive for %q: %v", importPath, err)
}
return nil // Cache miss.
}
defer f.Close()
a, err := compiler.ReadArchive(importPath, f)
if err != nil {
log.Warningf("Failed to read cached package archive for %q: %v", importPath, err)
return nil // Invalid/corrupted archive, cache miss.
}
log.Infof("Found cached package archive for %q, built at %v.", importPath, a.BuildTime)
return a
}

Expand Down
2 changes: 1 addition & 1 deletion compiler/gopherjspkg/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@ package gopherjspkg

import (
"go/build"
"log"
"net/http"
"os"
pathpkg "path"

"github.com/shurcooL/httpfs/filter"
log "github.com/sirupsen/logrus"
)

// FS is a virtual filesystem that contains core GopherJS packages.
Expand Down
2 changes: 1 addition & 1 deletion compiler/natives/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@ package natives

import (
"go/build"
"log"
"net/http"
"os"
"strings"

"github.com/shurcooL/httpfs/filter"
log "github.com/sirupsen/logrus"
)

// FS is a virtual filesystem that contains native packages.
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ require (
require (
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
)
3 changes: 3 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,8 @@ github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJ
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 h1:pXY9qYc/MP5zdvqWEUH6SjNiu7VhSjuVFTFiTcphaLU=
github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
Expand Down Expand Up @@ -365,6 +367,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand Down
27 changes: 25 additions & 2 deletions tool.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
"go/types"
"io"
"io/ioutil"
"log"
"net"
"net/http"
"os"
Expand All @@ -32,9 +31,11 @@ import (
"unicode/utf8"

gbuild "github.com/gopherjs/gopherjs/build"
"github.com/gopherjs/gopherjs/build/cache"
"github.com/gopherjs/gopherjs/compiler"
"github.com/gopherjs/gopherjs/internal/sysutil"
"github.com/neelance/sourcemap"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"golang.org/x/crypto/ssh/terminal"
Expand Down Expand Up @@ -85,6 +86,7 @@ func main() {
compilerFlags.BoolVar(&options.Color, "color", terminal.IsTerminal(int(os.Stderr.Fd())) && os.Getenv("TERM") != "dumb", "colored output")
compilerFlags.StringVar(&tags, "tags", "", "a list of build tags to consider satisfied during the build")
compilerFlags.BoolVar(&options.MapToLocalDisk, "localmap", false, "use local paths for sourcemap")
compilerFlags.BoolVarP(&options.NoCache, "no_cache", "a", false, "rebuild all packages from scratch")

flagWatch := pflag.NewFlagSet("", 0)
flagWatch.BoolVarP(&options.Watch, "watch", "w", false, "watch for changes to the source files")
Expand Down Expand Up @@ -590,11 +592,32 @@ func main() {
fmt.Printf("GopherJS %s\n", compiler.Version)
}

cmdClean := &cobra.Command{
Use: "clean",
Short: "clean GopherJS build cache",
}
cmdClean.RunE = func(cmd *cobra.Command, args []string) error {
return cache.Clear()
}

rootCmd := &cobra.Command{
Use: "gopherjs",
Long: "GopherJS is a tool for compiling Go source code to JavaScript.",
}
rootCmd.AddCommand(cmdBuild, cmdGet, cmdInstall, cmdRun, cmdTest, cmdServe, cmdVersion, cmdDoc)
rootCmd.AddCommand(cmdBuild, cmdGet, cmdInstall, cmdRun, cmdTest, cmdServe, cmdVersion, cmdDoc, cmdClean)

{
var logLevel string
rootCmd.PersistentFlags().StringVar(&logLevel, "log_level", log.ErrorLevel.String(), "Compiler log level (debug, info, warn, error, fatal, panic).")
rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {
lvl, err := log.ParseLevel(logLevel)
if err != nil {
return fmt.Errorf("invalid --log_level value %q: %w", logLevel, err)
}
log.SetLevel(lvl)
return nil
}
}
err := rootCmd.Execute()
if err != nil {
os.Exit(2)
Expand Down