From b8cb75b47ee594cadc7bec33da7b08452416a49b Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Mon, 23 Apr 2018 07:59:39 +0100 Subject: [PATCH 01/15] compiler: use hash calculation for determining staleness --- build/build.go | 96 +++++++++++++++++++++++--------------------- compiler/compiler.go | 1 + 2 files changed, 52 insertions(+), 45 deletions(-) diff --git a/build/build.go b/build/build.go index d9c0a2f42..b2ca54d78 100644 --- a/build/build.go +++ b/build/build.go @@ -6,6 +6,8 @@ package build import ( + "bytes" + "crypto/sha256" "fmt" "go/ast" "go/build" @@ -13,6 +15,7 @@ import ( "go/scanner" "go/token" "go/types" + "io" "io/ioutil" "os" "os/exec" @@ -20,7 +23,6 @@ import ( "path/filepath" "strconv" "strings" - "time" "github.com/fsnotify/fsnotify" "github.com/gopherjs/gopherjs/compiler" @@ -409,11 +411,10 @@ func (o *Options) PrintSuccess(format string, a ...interface{}) { // GopherJS requires. type PackageData struct { *build.Package - JSFiles []string - IsTest bool // IsTest is true if the package is being built for running tests. - SrcModTime time.Time - UpToDate bool - IsVirtual bool // If true, the package does not have a corresponding physical directory on disk. + JSFiles []string + IsTest bool // IsTest is true if the package is being built for running tests. + UpToDate bool + IsVirtual bool // If true, the package does not have a corresponding physical directory on disk. bctx *build.Context // The original build context this package came from. } @@ -593,19 +594,19 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { return archive, nil } + pkgHash := sha256.New() + if pkg.PkgObj != "" { - var fileInfo os.FileInfo - gopherjsBinary, err := os.Executable() - if err == nil { - fileInfo, err = os.Stat(gopherjsBinary) - if err == nil { - pkg.SrcModTime = fileInfo.ModTime() - } + binPath, err := os.Executable() + if err != nil { + return nil, fmt.Errorf("could not locate GopherJS binary: %v", err) } + binFile, err := os.Open(binPath) if err != nil { - os.Stderr.WriteString("Could not get GopherJS binary's modification timestamp. Please report issue.\n") - pkg.SrcModTime = time.Now() + return nil, fmt.Errorf("could not open %v: %v", binPath, err) } + defer binFile.Close() + io.Copy(pkgHash, binFile) for _, importedPkgPath := range pkg.Imports { // Ignore all imports that aren't mentioned in import specs of pkg. @@ -627,50 +628,57 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if importedPkgPath == "unsafe" || ignored { continue } - importedPkg, _, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir) + _, importedArchive, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir) if err != nil { return nil, err } - impModTime := importedPkg.SrcModTime - if impModTime.After(pkg.SrcModTime) { - pkg.SrcModTime = impModTime - } + + fmt.Fprintf(pkgHash, "import: %v\n", importedPkgPath) + fmt.Fprintf(pkgHash, " hash: %v\n", importedArchive.Hash) } for _, name := range append(pkg.GoFiles, pkg.JSFiles...) { - fileInfo, err := statFile(filepath.Join(pkg.Dir, name)) + fp := filepath.Join(pkg.Dir, name) + file, err := s.bctx.OpenFile(fp) if err != nil { - return nil, err - } - if fileInfo.ModTime().After(pkg.SrcModTime) { - pkg.SrcModTime = fileInfo.ModTime() + return nil, fmt.Errorf("failed to open %v: %v", fp, err) } + fmt.Fprintf(pkgHash, "file: %v\n", fp) + n, _ := io.Copy(pkgHash, file) + fmt.Fprintf(pkgHash, "%d bytes\n", n) } - pkgObjFileInfo, err := os.Stat(pkg.PkgObj) - if err == nil && !pkg.SrcModTime.After(pkgObjFileInfo.ModTime()) { - // package object is up to date, load from disk if library - pkg.UpToDate = true - if pkg.IsCommand() { - return nil, nil - } + // no commands are archived + if pkg.IsCommand() { + goto CacheMiss + } - objFile, err := os.Open(pkg.PkgObj) - if err != nil { - return nil, err + objFile, err := os.Open(pkg.PkgObj) + if err != nil { + if os.IsNotExist(err) { + goto CacheMiss } - defer objFile.Close() + return nil, err + } + defer objFile.Close() - archive, err := compiler.ReadArchive(pkg.PkgObj, pkg.ImportPath, objFile, s.Types) - if err != nil { - return nil, err - } + archive, err := compiler.ReadArchive(pkg.PkgObj, pkg.ImportPath, objFile, s.Types) + if err != nil { + return nil, err + } + if bytes.Equal(archive.Hash, pkgHash.Sum(nil)) { s.Archives[pkg.ImportPath] = archive - return archive, err + return archive, nil } } +CacheMiss: + + if s.options.Verbose { + fmt.Printf("Cache miss for %v\n", pkg.ImportPath) + } + fileSet := token.NewFileSet() files, err := parseAndAugment(s.xctx, pkg, pkg.IsTest, fileSet) if err != nil { @@ -697,6 +705,8 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { return nil, err } + archive.Hash = pkgHash.Sum(nil) + for _, jsFile := range pkg.JSFiles { code, err := ioutil.ReadFile(filepath.Join(pkg.Dir, jsFile)) if err != nil { @@ -707,10 +717,6 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { archive.IncJSCode = append(archive.IncJSCode, []byte("\n\t}).call($global);\n")...) } - if s.options.Verbose { - fmt.Println(pkg.ImportPath) - } - s.Archives[pkg.ImportPath] = archive if pkg.PkgObj == "" || pkg.IsCommand() { diff --git a/compiler/compiler.go b/compiler/compiler.go index ec38ed197..3c7a815dd 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -49,6 +49,7 @@ func (err ErrorList) Normalize() error { // // This is a logical equivalent of an object file in traditional compilers. type Archive struct { + Hash []byte // Package's full import path, e.g. "some/package/name". ImportPath string // Package's name as per "package" statement at the top of a source file. From b4e03df5a379914f44f92fc22820568eac3a5954 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Mon, 23 Apr 2018 08:31:27 +0100 Subject: [PATCH 02/15] Add --bv flag to test to log verbose build context messages --- tool.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tool.go b/tool.go index e3fde81a4..ff13b7558 100644 --- a/tool.go +++ b/tool.go @@ -317,13 +317,14 @@ func main() { run := cmdTest.Flags().String("run", "", "Run only those tests and examples matching the regular expression.") short := cmdTest.Flags().Bool("short", false, "Tell long-running tests to shorten their run time.") verbose := cmdTest.Flags().BoolP("verbose", "v", false, "Log all tests as they are run. Also print all text from Log and Logf calls even if the test succeeds.") + buildVerbose := cmdTest.Flags().BoolP("bv", "", false, "Use a verbose build context.") compileOnly := cmdTest.Flags().BoolP("compileonly", "c", false, "Compile the test binary to pkg.test.js but do not run it (where pkg is the last element of the package's import path). The file name can be changed with the -o flag.") outputFilename := cmdTest.Flags().StringP("output", "o", "", "Compile the test binary to the named file. The test still runs (unless -c is specified).") parallelTests := cmdTest.Flags().IntP("parallel", "p", runtime.NumCPU(), "Allow running tests in parallel for up to -p packages. Tests within the same package are still executed sequentially.") cmdTest.Flags().AddFlagSet(compilerFlags) cmdTest.Run = func(cmd *cobra.Command, args []string) { options.BuildTags = strings.Fields(tags) - + options.Verbose = *buildVerbose err := func() error { // Expand import path patterns. patternContext := gbuild.NewBuildContext("", options.BuildTags) From a3e7e93feb0fef819311e656f262c3ae86fd5ef8 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Mon, 23 Apr 2018 09:15:12 +0100 Subject: [PATCH 03/15] Add build tags to the hash --- build/build.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build/build.go b/build/build.go index b2ca54d78..754668219 100644 --- a/build/build.go +++ b/build/build.go @@ -21,6 +21,7 @@ import ( "os/exec" "path" "path/filepath" + "sort" "strconv" "strings" @@ -608,6 +609,11 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { defer binFile.Close() io.Copy(pkgHash, binFile) + orderedBuildTags := append([]string{}, s.options.BuildTags...) + sort.Strings(orderedBuildTags) + + fmt.Fprintf(pkgHash, "build tags: %v\n", strings.Join(orderedBuildTags, ",")) + for _, importedPkgPath := range pkg.Imports { // Ignore all imports that aren't mentioned in import specs of pkg. // For example, this ignores imports such as runtime/internal/sys and runtime/internal/atomic. From 2109552a69f8af4588e6eef55e7e1ebde807b0b3 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Mon, 23 Apr 2018 13:31:40 +0100 Subject: [PATCH 04/15] More readable version of hash for debug purposes --- build/build.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/build/build.go b/build/build.go index 754668219..d03cc56ab 100644 --- a/build/build.go +++ b/build/build.go @@ -607,7 +607,10 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { return nil, fmt.Errorf("could not open %v: %v", binPath, err) } defer binFile.Close() - io.Copy(pkgHash, binFile) + + binHash := sha256.New() + io.Copy(binHash, binFile) + fmt.Fprintf(pkgHash, "gopherjs bin: %#x\n", binHash.Sum(nil)) orderedBuildTags := append([]string{}, s.options.BuildTags...) sort.Strings(orderedBuildTags) @@ -640,7 +643,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { } fmt.Fprintf(pkgHash, "import: %v\n", importedPkgPath) - fmt.Fprintf(pkgHash, " hash: %v\n", importedArchive.Hash) + fmt.Fprintf(pkgHash, " hash: %#x\n", importedArchive.Hash) } for _, name := range append(pkg.GoFiles, pkg.JSFiles...) { From 1198293ad9adb63b7ef9467a03a9f744f2970955 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Mon, 23 Apr 2018 14:29:25 +0100 Subject: [PATCH 05/15] Fix file closing --- build/build.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build/build.go b/build/build.go index d03cc56ab..920605ea3 100644 --- a/build/build.go +++ b/build/build.go @@ -606,10 +606,10 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if err != nil { return nil, fmt.Errorf("could not open %v: %v", binPath, err) } - defer binFile.Close() binHash := sha256.New() io.Copy(binHash, binFile) + binFile.Close() fmt.Fprintf(pkgHash, "gopherjs bin: %#x\n", binHash.Sum(nil)) orderedBuildTags := append([]string{}, s.options.BuildTags...) @@ -654,6 +654,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { } fmt.Fprintf(pkgHash, "file: %v\n", fp) n, _ := io.Copy(pkgHash, file) + file.Close() fmt.Fprintf(pkgHash, "%d bytes\n", n) } @@ -669,9 +670,9 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { } return nil, err } - defer objFile.Close() archive, err := compiler.ReadArchive(pkg.PkgObj, pkg.ImportPath, objFile, s.Types) + objFile.Close() if err != nil { return nil, err } From 7c2a929d235905924567c7db5ed04a8bdb5a3160 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Mon, 23 Apr 2018 17:42:14 +0100 Subject: [PATCH 06/15] Address feedback from hajimehoshi --- build/build.go | 72 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/build/build.go b/build/build.go index 920605ea3..68e60cd74 100644 --- a/build/build.go +++ b/build/build.go @@ -590,14 +590,42 @@ func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*Packag return pkg, archive, nil } +const ( + hashDebug = false +) + func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if archive, ok := s.Archives[pkg.ImportPath]; ok { return archive, nil } + // For non-main and test packages we build up a hash that will help + // determine staleness. Set hashDebug to see this in action. The format is: + // + // ## sync + // gopherjs bin: 0x519d22c6ab65a950f5b6278e4d65cb75dbd3a7eb1cf16e976a40b9f1febc0446 + // build tags: + // import: internal/race + // hash: 0xb966d7680c1c8ca75026f993c153aff0102dc9551f314e5352043187b5f9c9a6 + // ... + // + // file: /home/myitcv/gos/src/sync/cond.go + // + // N bytes + // ... + pkgHash := sha256.New() + var hw io.Writer = pkgHash + var hashDebugOut *bytes.Buffer + if hashDebug { + hashDebugOut = new(bytes.Buffer) + hw = io.MultiWriter(hashDebugOut, pkgHash) + } if pkg.PkgObj != "" { + fmt.Fprintf(hw, "## %v\n", pkg.ImportPath) + + binHash := sha256.New() binPath, err := os.Executable() if err != nil { return nil, fmt.Errorf("could not locate GopherJS binary: %v", err) @@ -606,16 +634,16 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if err != nil { return nil, fmt.Errorf("could not open %v: %v", binPath, err) } - - binHash := sha256.New() - io.Copy(binHash, binFile) + if _, err := io.Copy(binHash, binFile); err != nil { + return nil, fmt.Errorf("failed to hash %v: %v", binPath, err) + } binFile.Close() - fmt.Fprintf(pkgHash, "gopherjs bin: %#x\n", binHash.Sum(nil)) + fmt.Fprintf(hw, "gopherjs bin: %#x\n", binHash.Sum(nil)) orderedBuildTags := append([]string{}, s.options.BuildTags...) sort.Strings(orderedBuildTags) - fmt.Fprintf(pkgHash, "build tags: %v\n", strings.Join(orderedBuildTags, ",")) + fmt.Fprintf(hw, "build tags: %v\n", strings.Join(orderedBuildTags, ",")) for _, importedPkgPath := range pkg.Imports { // Ignore all imports that aren't mentioned in import specs of pkg. @@ -642,20 +670,34 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { return nil, err } - fmt.Fprintf(pkgHash, "import: %v\n", importedPkgPath) - fmt.Fprintf(pkgHash, " hash: %#x\n", importedArchive.Hash) + fmt.Fprintf(hw, "import: %v\n", importedPkgPath) + fmt.Fprintf(hw, " hash: %#x\n", importedArchive.Hash) } for _, name := range append(pkg.GoFiles, pkg.JSFiles...) { - fp := filepath.Join(pkg.Dir, name) - file, err := s.bctx.OpenFile(fp) - if err != nil { - return nil, fmt.Errorf("failed to open %v: %v", fp, err) + hashFile := func() error { + fp := filepath.Join(pkg.Dir, name) + file, err := s.bctx.OpenFile(fp) + if err != nil { + return fmt.Errorf("failed to open %v: %v", fp, err) + } + defer file.Close() + fmt.Fprintf(hw, "file: %v\n", fp) + n, err := io.Copy(hw, file) + if err != nil { + return fmt.Errorf("failed to hash file contents: %v", err) + } + fmt.Fprintf(hw, "%d bytes\n", n) + return nil } - fmt.Fprintf(pkgHash, "file: %v\n", fp) - n, _ := io.Copy(pkgHash, file) - file.Close() - fmt.Fprintf(pkgHash, "%d bytes\n", n) + + if err := hashFile(); err != nil { + return nil, fmt.Errorf("failed to hash file %v: %v", name, err) + } + } + + if hashDebug { + fmt.Printf("%s", hashDebugOut.String()) } // no commands are archived From 9b28a71e5d8a636b0c0bababc640c37769bb7dd6 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Mon, 23 Apr 2018 18:26:12 +0100 Subject: [PATCH 07/15] Refactor to prefer defers for Close --- build/build.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/build.go b/build/build.go index 68e60cd74..c5359d145 100644 --- a/build/build.go +++ b/build/build.go @@ -634,10 +634,10 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if err != nil { return nil, fmt.Errorf("could not open %v: %v", binPath, err) } + defer binFile.Close() if _, err := io.Copy(binHash, binFile); err != nil { return nil, fmt.Errorf("failed to hash %v: %v", binPath, err) } - binFile.Close() fmt.Fprintf(hw, "gopherjs bin: %#x\n", binHash.Sum(nil)) orderedBuildTags := append([]string{}, s.options.BuildTags...) @@ -712,9 +712,9 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { } return nil, err } + defer objFile.Close() archive, err := compiler.ReadArchive(pkg.PkgObj, pkg.ImportPath, objFile, s.Types) - objFile.Close() if err != nil { return nil, err } From 4041566f386df4cfa08ad0df4930ad44c2df899f Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Mon, 23 Apr 2018 18:32:30 +0100 Subject: [PATCH 08/15] Further refactor to use defer for close --- build/build.go | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/build/build.go b/build/build.go index c5359d145..3b4463a06 100644 --- a/build/build.go +++ b/build/build.go @@ -625,20 +625,27 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if pkg.PkgObj != "" { fmt.Fprintf(hw, "## %v\n", pkg.ImportPath) - binHash := sha256.New() - binPath, err := os.Executable() - if err != nil { - return nil, fmt.Errorf("could not locate GopherJS binary: %v", err) - } - binFile, err := os.Open(binPath) - if err != nil { - return nil, fmt.Errorf("could not open %v: %v", binPath, err) + hashBin := func() error { + binHash := sha256.New() + binPath, err := os.Executable() + if err != nil { + return fmt.Errorf("could not locate GopherJS binary: %v", err) + } + binFile, err := os.Open(binPath) + if err != nil { + return fmt.Errorf("could not open %v: %v", binPath, err) + } + defer binFile.Close() + if _, err := io.Copy(binHash, binFile); err != nil { + return fmt.Errorf("failed to hash %v: %v", binPath, err) + } + fmt.Fprintf(hw, "gopherjs bin: %#x\n", binHash.Sum(nil)) + return nil } - defer binFile.Close() - if _, err := io.Copy(binHash, binFile); err != nil { - return nil, fmt.Errorf("failed to hash %v: %v", binPath, err) + + if err := hashBin(); err != nil { + return nil, err } - fmt.Fprintf(hw, "gopherjs bin: %#x\n", binHash.Sum(nil)) orderedBuildTags := append([]string{}, s.options.BuildTags...) sort.Strings(orderedBuildTags) From 71f27d5d168c51633a7dab8e33e803ec919d0ddc Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Mon, 23 Apr 2018 19:04:07 +0100 Subject: [PATCH 09/15] Use better key name for gopherjs binary hash --- build/build.go | 51 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/build/build.go b/build/build.go index 3b4463a06..bc6c8855d 100644 --- a/build/build.go +++ b/build/build.go @@ -594,6 +594,32 @@ const ( hashDebug = false ) +var ( + gopherJsBinaryHash string +) + +func hashGopherJsBinary() (string, error) { + if gopherJsBinaryHash != "" { + return gopherJsBinaryHash, nil + } + + binHash := sha256.New() + binPath, err := os.Executable() + if err != nil { + return "", fmt.Errorf("could not locate GopherJS binary: %v", err) + } + binFile, err := os.Open(binPath) + if err != nil { + return "", fmt.Errorf("could not open %v: %v", binPath, err) + } + defer binFile.Close() + if _, err := io.Copy(binHash, binFile); err != nil { + return "", fmt.Errorf("failed to hash %v: %v", binPath, err) + } + gopherJsBinaryHash = fmt.Sprintf("%#x", binHash.Sum(nil)) + return gopherJsBinaryHash, nil +} + func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if archive, ok := s.Archives[pkg.ImportPath]; ok { return archive, nil @@ -603,7 +629,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { // determine staleness. Set hashDebug to see this in action. The format is: // // ## sync - // gopherjs bin: 0x519d22c6ab65a950f5b6278e4d65cb75dbd3a7eb1cf16e976a40b9f1febc0446 + // gopherjs binary hash: 0x519d22c6ab65a950f5b6278e4d65cb75dbd3a7eb1cf16e976a40b9f1febc0446 // build tags: // import: internal/race // hash: 0xb966d7680c1c8ca75026f993c153aff0102dc9551f314e5352043187b5f9c9a6 @@ -625,28 +651,13 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if pkg.PkgObj != "" { fmt.Fprintf(hw, "## %v\n", pkg.ImportPath) - hashBin := func() error { - binHash := sha256.New() - binPath, err := os.Executable() - if err != nil { - return fmt.Errorf("could not locate GopherJS binary: %v", err) - } - binFile, err := os.Open(binPath) - if err != nil { - return fmt.Errorf("could not open %v: %v", binPath, err) - } - defer binFile.Close() - if _, err := io.Copy(binHash, binFile); err != nil { - return fmt.Errorf("failed to hash %v: %v", binPath, err) - } - fmt.Fprintf(hw, "gopherjs bin: %#x\n", binHash.Sum(nil)) - return nil - } - - if err := hashBin(); err != nil { + binHash, err := hashGopherJsBinary() + if err != nil { return nil, err } + fmt.Fprintf(hw, "gopherjs binary hash: %v\n", binHash) + orderedBuildTags := append([]string{}, s.options.BuildTags...) sort.Strings(orderedBuildTags) From 17fb61e45d881318d182ecf355a077e5bac7e5b7 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Tue, 24 Apr 2018 07:09:24 +0100 Subject: [PATCH 10/15] Move compiler hash calc to init phase --- build/build.go | 53 ++++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/build/build.go b/build/build.go index bc6c8855d..c5807876f 100644 --- a/build/build.go +++ b/build/build.go @@ -50,6 +50,23 @@ var DefaultGOROOT = func() string { return build.Default.GOROOT }() +const ( + hashDebug = false +) + +var compilerBinaryHash string + +func init() { + // We do this here because it will only fail in truly bad situations, i.e. + // machine running out of resources. We also panic if there is a problem + // because it's unlikely anything else will be useful/work + h, err := hashCompilerBinary() + if err != nil { + panic(err) + } + compilerBinaryHash = h +} + type ImportCError struct { pkgPath string } @@ -590,17 +607,9 @@ func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*Packag return pkg, archive, nil } -const ( - hashDebug = false -) - -var ( - gopherJsBinaryHash string -) - -func hashGopherJsBinary() (string, error) { - if gopherJsBinaryHash != "" { - return gopherJsBinaryHash, nil +func hashCompilerBinary() (string, error) { + if compilerBinaryHash != "" { + return compilerBinaryHash, nil } binHash := sha256.New() @@ -616,8 +625,8 @@ func hashGopherJsBinary() (string, error) { if _, err := io.Copy(binHash, binFile); err != nil { return "", fmt.Errorf("failed to hash %v: %v", binPath, err) } - gopherJsBinaryHash = fmt.Sprintf("%#x", binHash.Sum(nil)) - return gopherJsBinaryHash, nil + compilerBinaryHash = fmt.Sprintf("%#x", binHash.Sum(nil)) + return compilerBinaryHash, nil } func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { @@ -628,14 +637,14 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { // For non-main and test packages we build up a hash that will help // determine staleness. Set hashDebug to see this in action. The format is: // - // ## sync - // gopherjs binary hash: 0x519d22c6ab65a950f5b6278e4d65cb75dbd3a7eb1cf16e976a40b9f1febc0446 - // build tags: - // import: internal/race + // ## + // compiler binary hash: 0x519d22c6ab65a950f5b6278e4d65cb75dbd3a7eb1cf16e976a40b9f1febc0446 + // build tags: + // import: // hash: 0xb966d7680c1c8ca75026f993c153aff0102dc9551f314e5352043187b5f9c9a6 // ... // - // file: /home/myitcv/gos/src/sync/cond.go + // file: // // N bytes // ... @@ -650,13 +659,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if pkg.PkgObj != "" { fmt.Fprintf(hw, "## %v\n", pkg.ImportPath) - - binHash, err := hashGopherJsBinary() - if err != nil { - return nil, err - } - - fmt.Fprintf(hw, "gopherjs binary hash: %v\n", binHash) + fmt.Fprintf(hw, "compiler binary hash: %v\n", compilerBinaryHash) orderedBuildTags := append([]string{}, s.options.BuildTags...) sort.Strings(orderedBuildTags) From 10577aca381eb6765f14a7fcb11e4fcc7bf3af94 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Tue, 24 Apr 2018 13:48:47 +0100 Subject: [PATCH 11/15] Remove --bv flag on test --- tool.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/tool.go b/tool.go index ff13b7558..49abc2f92 100644 --- a/tool.go +++ b/tool.go @@ -317,14 +317,12 @@ func main() { run := cmdTest.Flags().String("run", "", "Run only those tests and examples matching the regular expression.") short := cmdTest.Flags().Bool("short", false, "Tell long-running tests to shorten their run time.") verbose := cmdTest.Flags().BoolP("verbose", "v", false, "Log all tests as they are run. Also print all text from Log and Logf calls even if the test succeeds.") - buildVerbose := cmdTest.Flags().BoolP("bv", "", false, "Use a verbose build context.") compileOnly := cmdTest.Flags().BoolP("compileonly", "c", false, "Compile the test binary to pkg.test.js but do not run it (where pkg is the last element of the package's import path). The file name can be changed with the -o flag.") outputFilename := cmdTest.Flags().StringP("output", "o", "", "Compile the test binary to the named file. The test still runs (unless -c is specified).") parallelTests := cmdTest.Flags().IntP("parallel", "p", runtime.NumCPU(), "Allow running tests in parallel for up to -p packages. Tests within the same package are still executed sequentially.") cmdTest.Flags().AddFlagSet(compilerFlags) cmdTest.Run = func(cmd *cobra.Command, args []string) { options.BuildTags = strings.Fields(tags) - options.Verbose = *buildVerbose err := func() error { // Expand import path patterns. patternContext := gbuild.NewBuildContext("", options.BuildTags) From a2b615d0090edafd5c9799f7675d2d5bef0a3cd2 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Sat, 28 Apr 2018 13:04:56 +0100 Subject: [PATCH 12/15] Add basic test of new hash staleness --- staleness_test.go | 197 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 staleness_test.go diff --git a/staleness_test.go b/staleness_test.go new file mode 100644 index 000000000..21c7bfa65 --- /dev/null +++ b/staleness_test.go @@ -0,0 +1,197 @@ +package main_test + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +func TestBasicHashStaleness(t *testing.T) { + defer func() { + err := recover() + if err != nil { + t.Fatalf("got an expected error: %v", err.(error)) + } + }() + + h := newHashTester(t) + + td := h.tempDir() + defer os.RemoveAll(td) + h.setEnv("GOPATH", td) + h.dir = h.mkdir(td, "src", "example.com", "rubbish") + h.mkdir(h.dir, "blah") + h.writeFile("main.go", ` + package main + import "example.com/rubbish/blah" + func main() { + print(blah.Name) + } + `) + h.writeFile(filepath.Join("blah", "blah.go"), ` + package blah + const Name = "blah" + `) + m := filepath.Join(td, "bin", "rubbish.js") + a := filepath.Join(td, "pkg", fmt.Sprintf("%v_js", runtime.GOOS), "example.com", "rubbish", "blah.a") + + // variables to hold the current (c) and new (n) archive (a) and main (m) + // os.FileInfos + var ca, cm, na, nm os.FileInfo + + // at this point neither main nor archive should exist + if h.statFile(m) != nil { + t.Fatalf("main %v existed when it shouldn't have", m) + } + if h.statFile(a) != nil { + t.Fatalf("archive %v existed when it shouldn't have", a) + } + + h.run("gopherjs", "install", "example.com/rubbish") + + // now both main and the archive should exist + ca = h.statFile(a) + if ca == nil { + t.Fatalf("archive %v should exist but doesn't", a) + } + cm = h.statFile(m) + if cm == nil { + t.Fatalf("main %v should exist but doesn't", a) + } + + // re-running the install will cause main to be rewritten; not the package archive + h.run("gopherjs", "install", "example.com/rubbish") + nm = h.statFile(m) + if !nm.ModTime().After(cm.ModTime()) { + t.Fatalf("expected to see modified main file %v; got %v; prev %v", m, nm.ModTime(), cm.ModTime()) + } + cm = nm + if na := h.statFile(a); !na.ModTime().Equal(ca.ModTime()) { + t.Fatalf("expected not to see modified archive file %v; got %v; want %v", a, na.ModTime(), ca.ModTime()) + } + + // touching the package file should have no effect on the archive + h.touch(filepath.Join("blah", "blah.go")) + h.run("gopherjs", "install", "example.com/rubbish/blah") // only install the package here + if na := h.statFile(a); !na.ModTime().Equal(ca.ModTime()) { + t.Fatalf("expected not to see modified archive file %v; got %v; want %v", a, na.ModTime(), ca.ModTime()) + } + + // now update package file - should cause modification time change + h.writeFile(filepath.Join("blah", "blah.go"), ` + package blah + const Name = "GopherJS" + `) + h.run("gopherjs", "install", "example.com/rubbish") + na = h.statFile(a) + if !na.ModTime().After(ca.ModTime()) { + t.Fatalf("expected to see modified archive file %v; got %v; prev %v", a, na.ModTime(), ca.ModTime()) + } + ca = na + + // now change build tags - should cause modification time change + h.run("gopherjs", "install", "--tags", "asdf", "example.com/rubbish") + na = h.statFile(a) + if !na.ModTime().After(ca.ModTime()) { + t.Fatalf("expected to see modified archive file %v; got %v; prev %v", a, na.ModTime(), ca.ModTime()) + } + ca = na +} + +type hashTester struct { + t *testing.T + dir string + env []string +} + +func newHashTester(t *testing.T) *hashTester { + wd, err := os.Getwd() + if err != nil { + fatalf("run failed to get working directory: %v", err) + } + return &hashTester{ + t: t, + dir: wd, + env: os.Environ(), + } +} + +func (h *hashTester) touch(path string) { + path = filepath.Join(h.dir, path) + now := time.Now().UTC() + if err := os.Chtimes(path, now, now); err != nil { + fatalf("failed to touch %v: %v", path, err) + } +} + +func (h *hashTester) statFile(path string) os.FileInfo { + fi, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + fatalf("failed to stat %v: %v", path, err) + } + + return fi +} + +func (h *hashTester) setEnv(key, val string) { + newEnv := []string{fmt.Sprintf("%v=%v", key, val)} + for _, e := range h.env { + if !strings.HasPrefix(e, key+"=") { + newEnv = append(newEnv, e) + } + } + h.env = newEnv +} + +func (h *hashTester) mkdir(dirs ...string) string { + d := filepath.Join(dirs...) + if err := os.MkdirAll(d, 0755); err != nil { + fatalf("failed to mkdir %v: %v\n", d, err) + } + return d +} + +func (h *hashTester) writeFile(path, contents string) { + path = filepath.Join(h.dir, path) + if err := ioutil.WriteFile(path, []byte(contents), 0644); err != nil { + fatalf("failed to write file %v: %v", path, err) + } +} + +func (h *hashTester) tempDir() string { + h.t.Helper() + + td, err := ioutil.TempDir("", "gopherjs_hashTester") + if err != nil { + fatalf("failed to create temp dir: %v", err) + } + + return td +} + +func (h *hashTester) run(c string, args ...string) { + h.t.Helper() + + cmd := exec.Command(c, args...) + cmd.Dir = h.dir + cmd.Env = h.env + + out, err := cmd.CombinedOutput() + if err != nil { + fullCmd := append([]string{c}, args...) + fatalf("failed to run %v: %v\n%v", strings.Join(fullCmd, " "), err, string(out)) + } +} + +func fatalf(format string, args ...interface{}) { + panic(fmt.Errorf(format, args...)) +} From a705e0d66aad2b0f5e76e65adb017600774b9e7b Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Tue, 28 Dec 2021 12:47:07 +0100 Subject: [PATCH 13/15] Adapt to current version of GopherJS --- build/build.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/build.go b/build/build.go index c5807876f..39e942ece 100644 --- a/build/build.go +++ b/build/build.go @@ -698,7 +698,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { for _, name := range append(pkg.GoFiles, pkg.JSFiles...) { hashFile := func() error { fp := filepath.Join(pkg.Dir, name) - file, err := s.bctx.OpenFile(fp) + file, err := buildutil.OpenFile(pkg.bctx, name) if err != nil { return fmt.Errorf("failed to open %v: %v", fp, err) } From 5a428b641f09b691742e536654758aeee28be144 Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Tue, 28 Dec 2021 13:14:57 +0100 Subject: [PATCH 14/15] Hash natives --- build/build.go | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/build/build.go b/build/build.go index 39e942ece..b5d973c50 100644 --- a/build/build.go +++ b/build/build.go @@ -8,6 +8,7 @@ package build import ( "bytes" "crypto/sha256" + "errors" "fmt" "go/ast" "go/build" @@ -240,6 +241,22 @@ func ImportDir(dir string, mode build.ImportMode, installSuffix string, buildTag return pkg, nil } +// newNativesContext returns a new nativesContext, with special considerations +// for the syscall package. +func newNativesContext(importPath string) *simpleCtx { + nativesContext := embeddedCtx(&withPrefix{fs: natives.FS, prefix: DefaultGOROOT}, "", nil) + + if importPath == "syscall" { + // Special handling for the syscall package, which uses OS native + // GOOS/GOARCH pair. This will no longer be necessary after + // https://github.com/gopherjs/gopherjs/issues/693. + nativesContext.bctx.GOARCH = build.Default.GOARCH + nativesContext.bctx.BuildTags = append(nativesContext.bctx.BuildTags, "js") + } + + return nativesContext +} + // parseAndAugment parses and returns all .go files of given pkg. // Standard Go library packages are augmented with files in compiler/natives folder. // If isTest is true and pkg.ImportPath has no _test suffix, package is built for running internal tests. @@ -262,15 +279,7 @@ func parseAndAugment(xctx XContext, pkg *PackageData, isTest bool, fileSet *toke importPath = importPath[:len(importPath)-5] } - nativesContext := embeddedCtx(&withPrefix{fs: natives.FS, prefix: DefaultGOROOT}, "", nil) - - if importPath == "syscall" { - // Special handling for the syscall package, which uses OS native - // GOOS/GOARCH pair. This will no longer be necessary after - // https://github.com/gopherjs/gopherjs/issues/693. - nativesContext.bctx.GOARCH = build.Default.GOARCH - nativesContext.bctx.BuildTags = append(nativesContext.bctx.BuildTags, "js") - } + nativesContext := newNativesContext(importPath) if nativesPkg, err := nativesContext.Import(importPath, "", 0); err == nil { names := nativesPkg.GoFiles @@ -699,6 +708,13 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { hashFile := func() error { fp := filepath.Join(pkg.Dir, name) file, err := buildutil.OpenFile(pkg.bctx, name) + if errors.Is(err, os.ErrNotExist) { + nativesCtx := newNativesContext(pkg.ImportPath) + if nativesPkg, e := nativesCtx.Import(pkg.ImportPath, "", 0); e == nil { + fullPath := path.Join(nativesPkg.Dir, name) + file, err = nativesCtx.bctx.OpenFile(fullPath) + } + } if err != nil { return fmt.Errorf("failed to open %v: %v", fp, err) } From e3d27da2a55c22e5d544a10c4802eeac9b834823 Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Tue, 28 Dec 2021 13:18:34 +0100 Subject: [PATCH 15/15] debug --- build/build.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build/build.go b/build/build.go index b5d973c50..65cec88f0 100644 --- a/build/build.go +++ b/build/build.go @@ -708,11 +708,16 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { hashFile := func() error { fp := filepath.Join(pkg.Dir, name) file, err := buildutil.OpenFile(pkg.bctx, name) + fmt.Printf("buildutil.OpenFile err: %s\n", err) if errors.Is(err, os.ErrNotExist) { + fmt.Printf("trying natives\n") nativesCtx := newNativesContext(pkg.ImportPath) if nativesPkg, e := nativesCtx.Import(pkg.ImportPath, "", 0); e == nil { fullPath := path.Join(nativesPkg.Dir, name) file, err = nativesCtx.bctx.OpenFile(fullPath) + fmt.Printf("nativesCtx.bctx.OpenFile err: %s\n", err) + } else { + fmt.Printf("nativesCtx.Import err: %s", e) } } if err != nil {