From e807fdbace0e13dcac5813adb331ffe3f9b1d1b0 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Mon, 23 Apr 2018 07:59:39 +0100 Subject: [PATCH 01/12] compiler: use hash calculation for determining staleness --- build/build.go | 95 +++++++++++++++++++++++--------------------- compiler/compiler.go | 1 + 2 files changed, 51 insertions(+), 45 deletions(-) diff --git a/build/build.go b/build/build.go index 82edb1f24..6e377e655 100644 --- a/build/build.go +++ b/build/build.go @@ -1,6 +1,8 @@ package build import ( + "bytes" + "crypto/sha256" "fmt" "go/ast" "go/build" @@ -17,7 +19,6 @@ import ( "runtime" "strconv" "strings" - "time" "github.com/fsnotify/fsnotify" "github.com/gopherjs/gopherjs/compiler" @@ -457,11 +458,10 @@ func (o *Options) PrintSuccess(format string, a ...interface{}) { type PackageData struct { *build.Package - JSFiles []string - IsTest bool // IsTest is true if the package is being built for running tests. - SrcModTime time.Time - UpToDate bool - IsVirtual bool // If true, the package does not have a corresponding physical directory on disk. + JSFiles []string + IsTest bool // IsTest is true if the package is being built for running tests. + UpToDate bool + IsVirtual bool // If true, the package does not have a corresponding physical directory on disk. } type Session struct { @@ -596,19 +596,19 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { return archive, nil } + pkgHash := sha256.New() + if pkg.PkgObj != "" { - var fileInfo os.FileInfo - gopherjsBinary, err := os.Executable() - if err == nil { - fileInfo, err = os.Stat(gopherjsBinary) - if err == nil { - pkg.SrcModTime = fileInfo.ModTime() - } + binPath, err := os.Executable() + if err != nil { + return nil, fmt.Errorf("could not locate GopherJS binary: %v", err) } + binFile, err := os.Open(binPath) if err != nil { - os.Stderr.WriteString("Could not get GopherJS binary's modification timestamp. Please report issue.\n") - pkg.SrcModTime = time.Now() + return nil, fmt.Errorf("could not open %v: %v", binPath, err) } + defer binFile.Close() + io.Copy(pkgHash, binFile) for _, importedPkgPath := range pkg.Imports { // Ignore all imports that aren't mentioned in import specs of pkg. @@ -630,50 +630,57 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if importedPkgPath == "unsafe" || ignored { continue } - importedPkg, _, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir) + _, importedArchive, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir) if err != nil { return nil, err } - impModTime := importedPkg.SrcModTime - if impModTime.After(pkg.SrcModTime) { - pkg.SrcModTime = impModTime - } + + fmt.Fprintf(pkgHash, "import: %v\n", importedPkgPath) + fmt.Fprintf(pkgHash, " hash: %v\n", importedArchive.Hash) } for _, name := range append(pkg.GoFiles, pkg.JSFiles...) { - fileInfo, err := statFile(filepath.Join(pkg.Dir, name)) + fp := filepath.Join(pkg.Dir, name) + file, err := s.bctx.OpenFile(fp) if err != nil { - return nil, err - } - if fileInfo.ModTime().After(pkg.SrcModTime) { - pkg.SrcModTime = fileInfo.ModTime() + return nil, fmt.Errorf("failed to open %v: %v", fp, err) } + fmt.Fprintf(pkgHash, "file: %v\n", fp) + n, _ := io.Copy(pkgHash, file) + fmt.Fprintf(pkgHash, "%d bytes\n", n) } - pkgObjFileInfo, err := os.Stat(pkg.PkgObj) - if err == nil && !pkg.SrcModTime.After(pkgObjFileInfo.ModTime()) { - // package object is up to date, load from disk if library - pkg.UpToDate = true - if pkg.IsCommand() { - return nil, nil - } + // no commands are archived + if pkg.IsCommand() { + goto CacheMiss + } - objFile, err := os.Open(pkg.PkgObj) - if err != nil { - return nil, err + objFile, err := os.Open(pkg.PkgObj) + if err != nil { + if os.IsNotExist(err) { + goto CacheMiss } - defer objFile.Close() + return nil, err + } + defer objFile.Close() - archive, err := compiler.ReadArchive(pkg.PkgObj, pkg.ImportPath, objFile, s.Types) - if err != nil { - return nil, err - } + archive, err := compiler.ReadArchive(pkg.PkgObj, pkg.ImportPath, objFile, s.Types) + if err != nil { + return nil, err + } + if bytes.Equal(archive.Hash, pkgHash.Sum(nil)) { s.Archives[pkg.ImportPath] = archive - return archive, err + return archive, nil } } +CacheMiss: + + if s.options.Verbose { + fmt.Printf("Cache miss for %v\n", pkg.ImportPath) + } + fileSet := token.NewFileSet() files, err := parseAndAugment(s.bctx, pkg.Package, pkg.IsTest, fileSet) if err != nil { @@ -700,6 +707,8 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { return nil, err } + archive.Hash = pkgHash.Sum(nil) + for _, jsFile := range pkg.JSFiles { code, err := ioutil.ReadFile(filepath.Join(pkg.Dir, jsFile)) if err != nil { @@ -710,10 +719,6 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { archive.IncJSCode = append(archive.IncJSCode, []byte("\n\t}).call($global);\n")...) } - if s.options.Verbose { - fmt.Println(pkg.ImportPath) - } - s.Archives[pkg.ImportPath] = archive if pkg.PkgObj == "" || pkg.IsCommand() { diff --git a/compiler/compiler.go b/compiler/compiler.go index c5828514e..b4756ee5a 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -32,6 +32,7 @@ func (err ErrorList) Error() string { } type Archive struct { + Hash []byte ImportPath string Name string Imports []string From e615455f73ba18fdd1ab515ba212c729fad99234 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Mon, 23 Apr 2018 08:31:27 +0100 Subject: [PATCH 02/12] Add --bv flag to test to log verbose build context messages --- tool.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tool.go b/tool.go index 4c580a139..0e08fc408 100644 --- a/tool.go +++ b/tool.go @@ -298,11 +298,13 @@ func main() { run := cmdTest.Flags().String("run", "", "Run only those tests and examples matching the regular expression.") short := cmdTest.Flags().Bool("short", false, "Tell long-running tests to shorten their run time.") verbose := cmdTest.Flags().BoolP("verbose", "v", false, "Log all tests as they are run. Also print all text from Log and Logf calls even if the test succeeds.") + buildVerbose := cmdTest.Flags().BoolP("bv", "", false, "Use a verbose build context.") compileOnly := cmdTest.Flags().BoolP("compileonly", "c", false, "Compile the test binary to pkg.test.js but do not run it (where pkg is the last element of the package's import path). The file name can be changed with the -o flag.") outputFilename := cmdTest.Flags().StringP("output", "o", "", "Compile the test binary to the named file. The test still runs (unless -c is specified).") cmdTest.Flags().AddFlagSet(compilerFlags) cmdTest.Run = func(cmd *cobra.Command, args []string) { options.BuildTags = strings.Fields(tags) + options.Verbose = *buildVerbose err := func() error { // Expand import path patterns. patternContext := gbuild.NewBuildContext("", options.BuildTags) From 69257f7911f57e59bffcef11880ca9b00cedba7b Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Mon, 23 Apr 2018 09:15:12 +0100 Subject: [PATCH 03/12] Add build tags to the hash --- build/build.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build/build.go b/build/build.go index 6e377e655..314f909d4 100644 --- a/build/build.go +++ b/build/build.go @@ -17,6 +17,7 @@ import ( "path" "path/filepath" "runtime" + "sort" "strconv" "strings" @@ -610,6 +611,11 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { defer binFile.Close() io.Copy(pkgHash, binFile) + orderedBuildTags := append([]string{}, s.options.BuildTags...) + sort.Strings(orderedBuildTags) + + fmt.Fprintf(pkgHash, "build tags: %v\n", strings.Join(orderedBuildTags, ",")) + for _, importedPkgPath := range pkg.Imports { // Ignore all imports that aren't mentioned in import specs of pkg. // For example, this ignores imports such as runtime/internal/sys and runtime/internal/atomic. From f6baa109ae44fec7040539794cdd6737f90ba397 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Mon, 23 Apr 2018 13:31:40 +0100 Subject: [PATCH 04/12] More readable version of hash for debug purposes --- build/build.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/build/build.go b/build/build.go index 314f909d4..51fe268ae 100644 --- a/build/build.go +++ b/build/build.go @@ -609,7 +609,10 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { return nil, fmt.Errorf("could not open %v: %v", binPath, err) } defer binFile.Close() - io.Copy(pkgHash, binFile) + + binHash := sha256.New() + io.Copy(binHash, binFile) + fmt.Fprintf(pkgHash, "gopherjs bin: %#x\n", binHash.Sum(nil)) orderedBuildTags := append([]string{}, s.options.BuildTags...) sort.Strings(orderedBuildTags) @@ -642,7 +645,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { } fmt.Fprintf(pkgHash, "import: %v\n", importedPkgPath) - fmt.Fprintf(pkgHash, " hash: %v\n", importedArchive.Hash) + fmt.Fprintf(pkgHash, " hash: %#x\n", importedArchive.Hash) } for _, name := range append(pkg.GoFiles, pkg.JSFiles...) { From 41c38df9de9cbee94e3cf402b9b13585722cd804 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Mon, 23 Apr 2018 14:29:25 +0100 Subject: [PATCH 05/12] Fix file closing --- build/build.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build/build.go b/build/build.go index 51fe268ae..bd8b099c9 100644 --- a/build/build.go +++ b/build/build.go @@ -608,10 +608,10 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if err != nil { return nil, fmt.Errorf("could not open %v: %v", binPath, err) } - defer binFile.Close() binHash := sha256.New() io.Copy(binHash, binFile) + binFile.Close() fmt.Fprintf(pkgHash, "gopherjs bin: %#x\n", binHash.Sum(nil)) orderedBuildTags := append([]string{}, s.options.BuildTags...) @@ -656,6 +656,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { } fmt.Fprintf(pkgHash, "file: %v\n", fp) n, _ := io.Copy(pkgHash, file) + file.Close() fmt.Fprintf(pkgHash, "%d bytes\n", n) } @@ -671,9 +672,9 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { } return nil, err } - defer objFile.Close() archive, err := compiler.ReadArchive(pkg.PkgObj, pkg.ImportPath, objFile, s.Types) + objFile.Close() if err != nil { return nil, err } From d73d4bbd142a4758f8d6a02e11b34f5069624cda Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Mon, 23 Apr 2018 17:42:14 +0100 Subject: [PATCH 06/12] Address feedback from hajimehoshi --- build/build.go | 72 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/build/build.go b/build/build.go index bd8b099c9..61125970d 100644 --- a/build/build.go +++ b/build/build.go @@ -592,14 +592,42 @@ func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*Packag return pkg, archive, nil } +const ( + hashDebug = false +) + func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if archive, ok := s.Archives[pkg.ImportPath]; ok { return archive, nil } + // For non-main and test packages we build up a hash that will help + // determine staleness. Set hashDebug to see this in action. The format is: + // + // ## sync + // gopherjs bin: 0x519d22c6ab65a950f5b6278e4d65cb75dbd3a7eb1cf16e976a40b9f1febc0446 + // build tags: + // import: internal/race + // hash: 0xb966d7680c1c8ca75026f993c153aff0102dc9551f314e5352043187b5f9c9a6 + // ... + // + // file: /home/myitcv/gos/src/sync/cond.go + // + // N bytes + // ... + pkgHash := sha256.New() + var hw io.Writer = pkgHash + var hashDebugOut *bytes.Buffer + if hashDebug { + hashDebugOut = new(bytes.Buffer) + hw = io.MultiWriter(hashDebugOut, pkgHash) + } if pkg.PkgObj != "" { + fmt.Fprintf(hw, "## %v\n", pkg.ImportPath) + + binHash := sha256.New() binPath, err := os.Executable() if err != nil { return nil, fmt.Errorf("could not locate GopherJS binary: %v", err) @@ -608,16 +636,16 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if err != nil { return nil, fmt.Errorf("could not open %v: %v", binPath, err) } - - binHash := sha256.New() - io.Copy(binHash, binFile) + if _, err := io.Copy(binHash, binFile); err != nil { + return nil, fmt.Errorf("failed to hash %v: %v", binPath, err) + } binFile.Close() - fmt.Fprintf(pkgHash, "gopherjs bin: %#x\n", binHash.Sum(nil)) + fmt.Fprintf(hw, "gopherjs bin: %#x\n", binHash.Sum(nil)) orderedBuildTags := append([]string{}, s.options.BuildTags...) sort.Strings(orderedBuildTags) - fmt.Fprintf(pkgHash, "build tags: %v\n", strings.Join(orderedBuildTags, ",")) + fmt.Fprintf(hw, "build tags: %v\n", strings.Join(orderedBuildTags, ",")) for _, importedPkgPath := range pkg.Imports { // Ignore all imports that aren't mentioned in import specs of pkg. @@ -644,20 +672,34 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { return nil, err } - fmt.Fprintf(pkgHash, "import: %v\n", importedPkgPath) - fmt.Fprintf(pkgHash, " hash: %#x\n", importedArchive.Hash) + fmt.Fprintf(hw, "import: %v\n", importedPkgPath) + fmt.Fprintf(hw, " hash: %#x\n", importedArchive.Hash) } for _, name := range append(pkg.GoFiles, pkg.JSFiles...) { - fp := filepath.Join(pkg.Dir, name) - file, err := s.bctx.OpenFile(fp) - if err != nil { - return nil, fmt.Errorf("failed to open %v: %v", fp, err) + hashFile := func() error { + fp := filepath.Join(pkg.Dir, name) + file, err := s.bctx.OpenFile(fp) + if err != nil { + return fmt.Errorf("failed to open %v: %v", fp, err) + } + defer file.Close() + fmt.Fprintf(hw, "file: %v\n", fp) + n, err := io.Copy(hw, file) + if err != nil { + return fmt.Errorf("failed to hash file contents: %v", err) + } + fmt.Fprintf(hw, "%d bytes\n", n) + return nil } - fmt.Fprintf(pkgHash, "file: %v\n", fp) - n, _ := io.Copy(pkgHash, file) - file.Close() - fmt.Fprintf(pkgHash, "%d bytes\n", n) + + if err := hashFile(); err != nil { + return nil, fmt.Errorf("failed to hash file %v: %v", name, err) + } + } + + if hashDebug { + fmt.Printf("%s", hashDebugOut.String()) } // no commands are archived From 9c06cba0629511fcdcc0131274e9a38be4adf622 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Mon, 23 Apr 2018 18:26:12 +0100 Subject: [PATCH 07/12] Refactor to prefer defers for Close --- build/build.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/build.go b/build/build.go index 61125970d..c76c4bf2c 100644 --- a/build/build.go +++ b/build/build.go @@ -636,10 +636,10 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if err != nil { return nil, fmt.Errorf("could not open %v: %v", binPath, err) } + defer binFile.Close() if _, err := io.Copy(binHash, binFile); err != nil { return nil, fmt.Errorf("failed to hash %v: %v", binPath, err) } - binFile.Close() fmt.Fprintf(hw, "gopherjs bin: %#x\n", binHash.Sum(nil)) orderedBuildTags := append([]string{}, s.options.BuildTags...) @@ -714,9 +714,9 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { } return nil, err } + defer objFile.Close() archive, err := compiler.ReadArchive(pkg.PkgObj, pkg.ImportPath, objFile, s.Types) - objFile.Close() if err != nil { return nil, err } From 82f8b60799bfbe60c8170dae3fe3567333c4acc5 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Mon, 23 Apr 2018 18:32:30 +0100 Subject: [PATCH 08/12] Further refactor to use defer for close --- build/build.go | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/build/build.go b/build/build.go index c76c4bf2c..a37dc964f 100644 --- a/build/build.go +++ b/build/build.go @@ -627,20 +627,27 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if pkg.PkgObj != "" { fmt.Fprintf(hw, "## %v\n", pkg.ImportPath) - binHash := sha256.New() - binPath, err := os.Executable() - if err != nil { - return nil, fmt.Errorf("could not locate GopherJS binary: %v", err) - } - binFile, err := os.Open(binPath) - if err != nil { - return nil, fmt.Errorf("could not open %v: %v", binPath, err) + hashBin := func() error { + binHash := sha256.New() + binPath, err := os.Executable() + if err != nil { + return fmt.Errorf("could not locate GopherJS binary: %v", err) + } + binFile, err := os.Open(binPath) + if err != nil { + return fmt.Errorf("could not open %v: %v", binPath, err) + } + defer binFile.Close() + if _, err := io.Copy(binHash, binFile); err != nil { + return fmt.Errorf("failed to hash %v: %v", binPath, err) + } + fmt.Fprintf(hw, "gopherjs bin: %#x\n", binHash.Sum(nil)) + return nil } - defer binFile.Close() - if _, err := io.Copy(binHash, binFile); err != nil { - return nil, fmt.Errorf("failed to hash %v: %v", binPath, err) + + if err := hashBin(); err != nil { + return nil, err } - fmt.Fprintf(hw, "gopherjs bin: %#x\n", binHash.Sum(nil)) orderedBuildTags := append([]string{}, s.options.BuildTags...) sort.Strings(orderedBuildTags) From 2e40249573de2aafe2a88d200ac517563c44d11e Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Mon, 23 Apr 2018 19:04:07 +0100 Subject: [PATCH 09/12] Use better key name for gopherjs binary hash --- build/build.go | 51 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/build/build.go b/build/build.go index a37dc964f..01a39250b 100644 --- a/build/build.go +++ b/build/build.go @@ -596,6 +596,32 @@ const ( hashDebug = false ) +var ( + gopherJsBinaryHash string +) + +func hashGopherJsBinary() (string, error) { + if gopherJsBinaryHash != "" { + return gopherJsBinaryHash, nil + } + + binHash := sha256.New() + binPath, err := os.Executable() + if err != nil { + return "", fmt.Errorf("could not locate GopherJS binary: %v", err) + } + binFile, err := os.Open(binPath) + if err != nil { + return "", fmt.Errorf("could not open %v: %v", binPath, err) + } + defer binFile.Close() + if _, err := io.Copy(binHash, binFile); err != nil { + return "", fmt.Errorf("failed to hash %v: %v", binPath, err) + } + gopherJsBinaryHash = fmt.Sprintf("%#x", binHash.Sum(nil)) + return gopherJsBinaryHash, nil +} + func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if archive, ok := s.Archives[pkg.ImportPath]; ok { return archive, nil @@ -605,7 +631,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { // determine staleness. Set hashDebug to see this in action. The format is: // // ## sync - // gopherjs bin: 0x519d22c6ab65a950f5b6278e4d65cb75dbd3a7eb1cf16e976a40b9f1febc0446 + // gopherjs binary hash: 0x519d22c6ab65a950f5b6278e4d65cb75dbd3a7eb1cf16e976a40b9f1febc0446 // build tags: // import: internal/race // hash: 0xb966d7680c1c8ca75026f993c153aff0102dc9551f314e5352043187b5f9c9a6 @@ -627,28 +653,13 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if pkg.PkgObj != "" { fmt.Fprintf(hw, "## %v\n", pkg.ImportPath) - hashBin := func() error { - binHash := sha256.New() - binPath, err := os.Executable() - if err != nil { - return fmt.Errorf("could not locate GopherJS binary: %v", err) - } - binFile, err := os.Open(binPath) - if err != nil { - return fmt.Errorf("could not open %v: %v", binPath, err) - } - defer binFile.Close() - if _, err := io.Copy(binHash, binFile); err != nil { - return fmt.Errorf("failed to hash %v: %v", binPath, err) - } - fmt.Fprintf(hw, "gopherjs bin: %#x\n", binHash.Sum(nil)) - return nil - } - - if err := hashBin(); err != nil { + binHash, err := hashGopherJsBinary() + if err != nil { return nil, err } + fmt.Fprintf(hw, "gopherjs binary hash: %v\n", binHash) + orderedBuildTags := append([]string{}, s.options.BuildTags...) sort.Strings(orderedBuildTags) From 3d5665a12bb1298f0fa5e9b85caf7cec64d78926 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Tue, 24 Apr 2018 07:09:24 +0100 Subject: [PATCH 10/12] Move compiler hash calc to init phase --- build/build.go | 55 +++++++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/build/build.go b/build/build.go index 01a39250b..ffd8655f0 100644 --- a/build/build.go +++ b/build/build.go @@ -30,6 +30,25 @@ import ( "golang.org/x/tools/go/buildutil" ) +const ( + hashDebug = false +) + +var ( + compilerBinaryHash string +) + +func init() { + // We do this here because it will only fail in truly bad situations, i.e. + // machine running out of resources. We also panic if there is a problem + // because it's unlikely anything else will be useful/work + h, err := hashCompilerBinary() + if err != nil { + panic(err) + } + compilerBinaryHash = h +} + type ImportCError struct { pkgPath string } @@ -592,17 +611,9 @@ func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*Packag return pkg, archive, nil } -const ( - hashDebug = false -) - -var ( - gopherJsBinaryHash string -) - -func hashGopherJsBinary() (string, error) { - if gopherJsBinaryHash != "" { - return gopherJsBinaryHash, nil +func hashCompilerBinary() (string, error) { + if compilerBinaryHash != "" { + return compilerBinaryHash, nil } binHash := sha256.New() @@ -618,8 +629,8 @@ func hashGopherJsBinary() (string, error) { if _, err := io.Copy(binHash, binFile); err != nil { return "", fmt.Errorf("failed to hash %v: %v", binPath, err) } - gopherJsBinaryHash = fmt.Sprintf("%#x", binHash.Sum(nil)) - return gopherJsBinaryHash, nil + compilerBinaryHash = fmt.Sprintf("%#x", binHash.Sum(nil)) + return compilerBinaryHash, nil } func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { @@ -630,14 +641,14 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { // For non-main and test packages we build up a hash that will help // determine staleness. Set hashDebug to see this in action. The format is: // - // ## sync - // gopherjs binary hash: 0x519d22c6ab65a950f5b6278e4d65cb75dbd3a7eb1cf16e976a40b9f1febc0446 - // build tags: - // import: internal/race + // ## + // compiler binary hash: 0x519d22c6ab65a950f5b6278e4d65cb75dbd3a7eb1cf16e976a40b9f1febc0446 + // build tags: + // import: // hash: 0xb966d7680c1c8ca75026f993c153aff0102dc9551f314e5352043187b5f9c9a6 // ... // - // file: /home/myitcv/gos/src/sync/cond.go + // file: // // N bytes // ... @@ -652,13 +663,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if pkg.PkgObj != "" { fmt.Fprintf(hw, "## %v\n", pkg.ImportPath) - - binHash, err := hashGopherJsBinary() - if err != nil { - return nil, err - } - - fmt.Fprintf(hw, "gopherjs binary hash: %v\n", binHash) + fmt.Fprintf(hw, "compiler binary hash: %v\n", compilerBinaryHash) orderedBuildTags := append([]string{}, s.options.BuildTags...) sort.Strings(orderedBuildTags) From 5c358e33ac9499a15242122217cafbba60b7d089 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Tue, 24 Apr 2018 13:48:47 +0100 Subject: [PATCH 11/12] Remove --bv flag on test --- tool.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/tool.go b/tool.go index 0e08fc408..4c580a139 100644 --- a/tool.go +++ b/tool.go @@ -298,13 +298,11 @@ func main() { run := cmdTest.Flags().String("run", "", "Run only those tests and examples matching the regular expression.") short := cmdTest.Flags().Bool("short", false, "Tell long-running tests to shorten their run time.") verbose := cmdTest.Flags().BoolP("verbose", "v", false, "Log all tests as they are run. Also print all text from Log and Logf calls even if the test succeeds.") - buildVerbose := cmdTest.Flags().BoolP("bv", "", false, "Use a verbose build context.") compileOnly := cmdTest.Flags().BoolP("compileonly", "c", false, "Compile the test binary to pkg.test.js but do not run it (where pkg is the last element of the package's import path). The file name can be changed with the -o flag.") outputFilename := cmdTest.Flags().StringP("output", "o", "", "Compile the test binary to the named file. The test still runs (unless -c is specified).") cmdTest.Flags().AddFlagSet(compilerFlags) cmdTest.Run = func(cmd *cobra.Command, args []string) { options.BuildTags = strings.Fields(tags) - options.Verbose = *buildVerbose err := func() error { // Expand import path patterns. patternContext := gbuild.NewBuildContext("", options.BuildTags) From 1ed4e6ae41d98e9990e019d960aa6b4a573fbc04 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Sat, 28 Apr 2018 13:04:56 +0100 Subject: [PATCH 12/12] Add basic test of new hash staleness --- staleness_test.go | 197 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 staleness_test.go diff --git a/staleness_test.go b/staleness_test.go new file mode 100644 index 000000000..21c7bfa65 --- /dev/null +++ b/staleness_test.go @@ -0,0 +1,197 @@ +package main_test + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +func TestBasicHashStaleness(t *testing.T) { + defer func() { + err := recover() + if err != nil { + t.Fatalf("got an expected error: %v", err.(error)) + } + }() + + h := newHashTester(t) + + td := h.tempDir() + defer os.RemoveAll(td) + h.setEnv("GOPATH", td) + h.dir = h.mkdir(td, "src", "example.com", "rubbish") + h.mkdir(h.dir, "blah") + h.writeFile("main.go", ` + package main + import "example.com/rubbish/blah" + func main() { + print(blah.Name) + } + `) + h.writeFile(filepath.Join("blah", "blah.go"), ` + package blah + const Name = "blah" + `) + m := filepath.Join(td, "bin", "rubbish.js") + a := filepath.Join(td, "pkg", fmt.Sprintf("%v_js", runtime.GOOS), "example.com", "rubbish", "blah.a") + + // variables to hold the current (c) and new (n) archive (a) and main (m) + // os.FileInfos + var ca, cm, na, nm os.FileInfo + + // at this point neither main nor archive should exist + if h.statFile(m) != nil { + t.Fatalf("main %v existed when it shouldn't have", m) + } + if h.statFile(a) != nil { + t.Fatalf("archive %v existed when it shouldn't have", a) + } + + h.run("gopherjs", "install", "example.com/rubbish") + + // now both main and the archive should exist + ca = h.statFile(a) + if ca == nil { + t.Fatalf("archive %v should exist but doesn't", a) + } + cm = h.statFile(m) + if cm == nil { + t.Fatalf("main %v should exist but doesn't", a) + } + + // re-running the install will cause main to be rewritten; not the package archive + h.run("gopherjs", "install", "example.com/rubbish") + nm = h.statFile(m) + if !nm.ModTime().After(cm.ModTime()) { + t.Fatalf("expected to see modified main file %v; got %v; prev %v", m, nm.ModTime(), cm.ModTime()) + } + cm = nm + if na := h.statFile(a); !na.ModTime().Equal(ca.ModTime()) { + t.Fatalf("expected not to see modified archive file %v; got %v; want %v", a, na.ModTime(), ca.ModTime()) + } + + // touching the package file should have no effect on the archive + h.touch(filepath.Join("blah", "blah.go")) + h.run("gopherjs", "install", "example.com/rubbish/blah") // only install the package here + if na := h.statFile(a); !na.ModTime().Equal(ca.ModTime()) { + t.Fatalf("expected not to see modified archive file %v; got %v; want %v", a, na.ModTime(), ca.ModTime()) + } + + // now update package file - should cause modification time change + h.writeFile(filepath.Join("blah", "blah.go"), ` + package blah + const Name = "GopherJS" + `) + h.run("gopherjs", "install", "example.com/rubbish") + na = h.statFile(a) + if !na.ModTime().After(ca.ModTime()) { + t.Fatalf("expected to see modified archive file %v; got %v; prev %v", a, na.ModTime(), ca.ModTime()) + } + ca = na + + // now change build tags - should cause modification time change + h.run("gopherjs", "install", "--tags", "asdf", "example.com/rubbish") + na = h.statFile(a) + if !na.ModTime().After(ca.ModTime()) { + t.Fatalf("expected to see modified archive file %v; got %v; prev %v", a, na.ModTime(), ca.ModTime()) + } + ca = na +} + +type hashTester struct { + t *testing.T + dir string + env []string +} + +func newHashTester(t *testing.T) *hashTester { + wd, err := os.Getwd() + if err != nil { + fatalf("run failed to get working directory: %v", err) + } + return &hashTester{ + t: t, + dir: wd, + env: os.Environ(), + } +} + +func (h *hashTester) touch(path string) { + path = filepath.Join(h.dir, path) + now := time.Now().UTC() + if err := os.Chtimes(path, now, now); err != nil { + fatalf("failed to touch %v: %v", path, err) + } +} + +func (h *hashTester) statFile(path string) os.FileInfo { + fi, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + fatalf("failed to stat %v: %v", path, err) + } + + return fi +} + +func (h *hashTester) setEnv(key, val string) { + newEnv := []string{fmt.Sprintf("%v=%v", key, val)} + for _, e := range h.env { + if !strings.HasPrefix(e, key+"=") { + newEnv = append(newEnv, e) + } + } + h.env = newEnv +} + +func (h *hashTester) mkdir(dirs ...string) string { + d := filepath.Join(dirs...) + if err := os.MkdirAll(d, 0755); err != nil { + fatalf("failed to mkdir %v: %v\n", d, err) + } + return d +} + +func (h *hashTester) writeFile(path, contents string) { + path = filepath.Join(h.dir, path) + if err := ioutil.WriteFile(path, []byte(contents), 0644); err != nil { + fatalf("failed to write file %v: %v", path, err) + } +} + +func (h *hashTester) tempDir() string { + h.t.Helper() + + td, err := ioutil.TempDir("", "gopherjs_hashTester") + if err != nil { + fatalf("failed to create temp dir: %v", err) + } + + return td +} + +func (h *hashTester) run(c string, args ...string) { + h.t.Helper() + + cmd := exec.Command(c, args...) + cmd.Dir = h.dir + cmd.Env = h.env + + out, err := cmd.CombinedOutput() + if err != nil { + fullCmd := append([]string{c}, args...) + fatalf("failed to run %v: %v\n%v", strings.Join(fullCmd, " "), err, string(out)) + } +} + +func fatalf(format string, args ...interface{}) { + panic(fmt.Errorf(format, args...)) +}