diff --git a/build/build.go b/build/build.go index d9c0a2f42..65cec88f0 100644 --- a/build/build.go +++ b/build/build.go @@ -6,6 +6,9 @@ package build import ( + "bytes" + "crypto/sha256" + "errors" "fmt" "go/ast" "go/build" @@ -13,14 +16,15 @@ import ( "go/scanner" "go/token" "go/types" + "io" "io/ioutil" "os" "os/exec" "path" "path/filepath" + "sort" "strconv" "strings" - "time" "github.com/fsnotify/fsnotify" "github.com/gopherjs/gopherjs/compiler" @@ -47,6 +51,23 @@ var DefaultGOROOT = func() string { return build.Default.GOROOT }() +const ( + hashDebug = false +) + +var compilerBinaryHash string + +func init() { + // We do this here because it will only fail in truly bad situations, i.e. + // machine running out of resources. We also panic if there is a problem + // because it's unlikely anything else will be useful/work + h, err := hashCompilerBinary() + if err != nil { + panic(err) + } + compilerBinaryHash = h +} + type ImportCError struct { pkgPath string } @@ -220,6 +241,22 @@ func ImportDir(dir string, mode build.ImportMode, installSuffix string, buildTag return pkg, nil } +// newNativesContext returns a new nativesContext, with special considerations +// for the syscall package. +func newNativesContext(importPath string) *simpleCtx { + nativesContext := embeddedCtx(&withPrefix{fs: natives.FS, prefix: DefaultGOROOT}, "", nil) + + if importPath == "syscall" { + // Special handling for the syscall package, which uses OS native + // GOOS/GOARCH pair. This will no longer be necessary after + // https://github.com/gopherjs/gopherjs/issues/693. + nativesContext.bctx.GOARCH = build.Default.GOARCH + nativesContext.bctx.BuildTags = append(nativesContext.bctx.BuildTags, "js") + } + + return nativesContext +} + // parseAndAugment parses and returns all .go files of given pkg. // Standard Go library packages are augmented with files in compiler/natives folder. // If isTest is true and pkg.ImportPath has no _test suffix, package is built for running internal tests. @@ -242,15 +279,7 @@ func parseAndAugment(xctx XContext, pkg *PackageData, isTest bool, fileSet *toke importPath = importPath[:len(importPath)-5] } - nativesContext := embeddedCtx(&withPrefix{fs: natives.FS, prefix: DefaultGOROOT}, "", nil) - - if importPath == "syscall" { - // Special handling for the syscall package, which uses OS native - // GOOS/GOARCH pair. This will no longer be necessary after - // https://github.com/gopherjs/gopherjs/issues/693. - nativesContext.bctx.GOARCH = build.Default.GOARCH - nativesContext.bctx.BuildTags = append(nativesContext.bctx.BuildTags, "js") - } + nativesContext := newNativesContext(importPath) if nativesPkg, err := nativesContext.Import(importPath, "", 0); err == nil { names := nativesPkg.GoFiles @@ -409,11 +438,10 @@ func (o *Options) PrintSuccess(format string, a ...interface{}) { // GopherJS requires. type PackageData struct { *build.Package - JSFiles []string - IsTest bool // IsTest is true if the package is being built for running tests. - SrcModTime time.Time - UpToDate bool - IsVirtual bool // If true, the package does not have a corresponding physical directory on disk. + JSFiles []string + IsTest bool // IsTest is true if the package is being built for running tests. + UpToDate bool + IsVirtual bool // If true, the package does not have a corresponding physical directory on disk. bctx *build.Context // The original build context this package came from. } @@ -588,24 +616,64 @@ func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*Packag return pkg, archive, nil } +func hashCompilerBinary() (string, error) { + if compilerBinaryHash != "" { + return compilerBinaryHash, nil + } + + binHash := sha256.New() + binPath, err := os.Executable() + if err != nil { + return "", fmt.Errorf("could not locate GopherJS binary: %v", err) + } + binFile, err := os.Open(binPath) + if err != nil { + return "", fmt.Errorf("could not open %v: %v", binPath, err) + } + defer binFile.Close() + if _, err := io.Copy(binHash, binFile); err != nil { + return "", fmt.Errorf("failed to hash %v: %v", binPath, err) + } + compilerBinaryHash = fmt.Sprintf("%#x", binHash.Sum(nil)) + return compilerBinaryHash, nil +} + func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if archive, ok := s.Archives[pkg.ImportPath]; ok { return archive, nil } + // For non-main and test packages we build up a hash that will help + // determine staleness. Set hashDebug to see this in action. The format is: + // + // ## + // compiler binary hash: 0x519d22c6ab65a950f5b6278e4d65cb75dbd3a7eb1cf16e976a40b9f1febc0446 + // build tags: + // import: + // hash: 0xb966d7680c1c8ca75026f993c153aff0102dc9551f314e5352043187b5f9c9a6 + // ... + // + // file: + // + // N bytes + // ... + + pkgHash := sha256.New() + var hw io.Writer = pkgHash + var hashDebugOut *bytes.Buffer + if hashDebug { + hashDebugOut = new(bytes.Buffer) + hw = io.MultiWriter(hashDebugOut, pkgHash) + } + if pkg.PkgObj != "" { - var fileInfo os.FileInfo - gopherjsBinary, err := os.Executable() - if err == nil { - fileInfo, err = os.Stat(gopherjsBinary) - if err == nil { - pkg.SrcModTime = fileInfo.ModTime() - } - } - if err != nil { - os.Stderr.WriteString("Could not get GopherJS binary's modification timestamp. Please report issue.\n") - pkg.SrcModTime = time.Now() - } + fmt.Fprintf(hw, "## %v\n", pkg.ImportPath) + fmt.Fprintf(hw, "compiler binary hash: %v\n", compilerBinaryHash) + + orderedBuildTags := append([]string{}, s.options.BuildTags...) + sort.Strings(orderedBuildTags) + + fmt.Fprintf(hw, "build tags: %v\n", strings.Join(orderedBuildTags, ",")) for _, importedPkgPath := range pkg.Imports { // Ignore all imports that aren't mentioned in import specs of pkg. @@ -627,50 +695,84 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if importedPkgPath == "unsafe" || ignored { continue } - importedPkg, _, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir) + _, importedArchive, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir) if err != nil { return nil, err } - impModTime := importedPkg.SrcModTime - if impModTime.After(pkg.SrcModTime) { - pkg.SrcModTime = impModTime - } + + fmt.Fprintf(hw, "import: %v\n", importedPkgPath) + fmt.Fprintf(hw, " hash: %#x\n", importedArchive.Hash) } for _, name := range append(pkg.GoFiles, pkg.JSFiles...) { - fileInfo, err := statFile(filepath.Join(pkg.Dir, name)) - if err != nil { - return nil, err + hashFile := func() error { + fp := filepath.Join(pkg.Dir, name) + file, err := buildutil.OpenFile(pkg.bctx, name) + fmt.Printf("buildutil.OpenFile err: %s\n", err) + if errors.Is(err, os.ErrNotExist) { + fmt.Printf("trying natives\n") + nativesCtx := newNativesContext(pkg.ImportPath) + if nativesPkg, e := nativesCtx.Import(pkg.ImportPath, "", 0); e == nil { + fullPath := path.Join(nativesPkg.Dir, name) + file, err = nativesCtx.bctx.OpenFile(fullPath) + fmt.Printf("nativesCtx.bctx.OpenFile err: %s\n", err) + } else { + fmt.Printf("nativesCtx.Import err: %s", e) + } + } + if err != nil { + return fmt.Errorf("failed to open %v: %v", fp, err) + } + defer file.Close() + fmt.Fprintf(hw, "file: %v\n", fp) + n, err := io.Copy(hw, file) + if err != nil { + return fmt.Errorf("failed to hash file contents: %v", err) + } + fmt.Fprintf(hw, "%d bytes\n", n) + return nil } - if fileInfo.ModTime().After(pkg.SrcModTime) { - pkg.SrcModTime = fileInfo.ModTime() + + if err := hashFile(); err != nil { + return nil, fmt.Errorf("failed to hash file %v: %v", name, err) } } - pkgObjFileInfo, err := os.Stat(pkg.PkgObj) - if err == nil && !pkg.SrcModTime.After(pkgObjFileInfo.ModTime()) { - // package object is up to date, load from disk if library - pkg.UpToDate = true - if pkg.IsCommand() { - return nil, nil - } + if hashDebug { + fmt.Printf("%s", hashDebugOut.String()) + } - objFile, err := os.Open(pkg.PkgObj) - if err != nil { - return nil, err - } - defer objFile.Close() + // no commands are archived + if pkg.IsCommand() { + goto CacheMiss + } - archive, err := compiler.ReadArchive(pkg.PkgObj, pkg.ImportPath, objFile, s.Types) - if err != nil { - return nil, err + objFile, err := os.Open(pkg.PkgObj) + if err != nil { + if os.IsNotExist(err) { + goto CacheMiss } + return nil, err + } + defer objFile.Close() + + archive, err := compiler.ReadArchive(pkg.PkgObj, pkg.ImportPath, objFile, s.Types) + if err != nil { + return nil, err + } + if bytes.Equal(archive.Hash, pkgHash.Sum(nil)) { s.Archives[pkg.ImportPath] = archive - return archive, err + return archive, nil } } +CacheMiss: + + if s.options.Verbose { + fmt.Printf("Cache miss for %v\n", pkg.ImportPath) + } + fileSet := token.NewFileSet() files, err := parseAndAugment(s.xctx, pkg, pkg.IsTest, fileSet) if err != nil { @@ -697,6 +799,8 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { return nil, err } + archive.Hash = pkgHash.Sum(nil) + for _, jsFile := range pkg.JSFiles { code, err := ioutil.ReadFile(filepath.Join(pkg.Dir, jsFile)) if err != nil { @@ -707,10 +811,6 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { archive.IncJSCode = append(archive.IncJSCode, []byte("\n\t}).call($global);\n")...) } - if s.options.Verbose { - fmt.Println(pkg.ImportPath) - } - s.Archives[pkg.ImportPath] = archive if pkg.PkgObj == "" || pkg.IsCommand() { diff --git a/compiler/compiler.go b/compiler/compiler.go index ec38ed197..3c7a815dd 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -49,6 +49,7 @@ func (err ErrorList) Normalize() error { // // This is a logical equivalent of an object file in traditional compilers. type Archive struct { + Hash []byte // Package's full import path, e.g. "some/package/name". ImportPath string // Package's name as per "package" statement at the top of a source file. diff --git a/staleness_test.go b/staleness_test.go new file mode 100644 index 000000000..21c7bfa65 --- /dev/null +++ b/staleness_test.go @@ -0,0 +1,197 @@ +package main_test + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +func TestBasicHashStaleness(t *testing.T) { + defer func() { + err := recover() + if err != nil { + t.Fatalf("got an expected error: %v", err.(error)) + } + }() + + h := newHashTester(t) + + td := h.tempDir() + defer os.RemoveAll(td) + h.setEnv("GOPATH", td) + h.dir = h.mkdir(td, "src", "example.com", "rubbish") + h.mkdir(h.dir, "blah") + h.writeFile("main.go", ` + package main + import "example.com/rubbish/blah" + func main() { + print(blah.Name) + } + `) + h.writeFile(filepath.Join("blah", "blah.go"), ` + package blah + const Name = "blah" + `) + m := filepath.Join(td, "bin", "rubbish.js") + a := filepath.Join(td, "pkg", fmt.Sprintf("%v_js", runtime.GOOS), "example.com", "rubbish", "blah.a") + + // variables to hold the current (c) and new (n) archive (a) and main (m) + // os.FileInfos + var ca, cm, na, nm os.FileInfo + + // at this point neither main nor archive should exist + if h.statFile(m) != nil { + t.Fatalf("main %v existed when it shouldn't have", m) + } + if h.statFile(a) != nil { + t.Fatalf("archive %v existed when it shouldn't have", a) + } + + h.run("gopherjs", "install", "example.com/rubbish") + + // now both main and the archive should exist + ca = h.statFile(a) + if ca == nil { + t.Fatalf("archive %v should exist but doesn't", a) + } + cm = h.statFile(m) + if cm == nil { + t.Fatalf("main %v should exist but doesn't", a) + } + + // re-running the install will cause main to be rewritten; not the package archive + h.run("gopherjs", "install", "example.com/rubbish") + nm = h.statFile(m) + if !nm.ModTime().After(cm.ModTime()) { + t.Fatalf("expected to see modified main file %v; got %v; prev %v", m, nm.ModTime(), cm.ModTime()) + } + cm = nm + if na := h.statFile(a); !na.ModTime().Equal(ca.ModTime()) { + t.Fatalf("expected not to see modified archive file %v; got %v; want %v", a, na.ModTime(), ca.ModTime()) + } + + // touching the package file should have no effect on the archive + h.touch(filepath.Join("blah", "blah.go")) + h.run("gopherjs", "install", "example.com/rubbish/blah") // only install the package here + if na := h.statFile(a); !na.ModTime().Equal(ca.ModTime()) { + t.Fatalf("expected not to see modified archive file %v; got %v; want %v", a, na.ModTime(), ca.ModTime()) + } + + // now update package file - should cause modification time change + h.writeFile(filepath.Join("blah", "blah.go"), ` + package blah + const Name = "GopherJS" + `) + h.run("gopherjs", "install", "example.com/rubbish") + na = h.statFile(a) + if !na.ModTime().After(ca.ModTime()) { + t.Fatalf("expected to see modified archive file %v; got %v; prev %v", a, na.ModTime(), ca.ModTime()) + } + ca = na + + // now change build tags - should cause modification time change + h.run("gopherjs", "install", "--tags", "asdf", "example.com/rubbish") + na = h.statFile(a) + if !na.ModTime().After(ca.ModTime()) { + t.Fatalf("expected to see modified archive file %v; got %v; prev %v", a, na.ModTime(), ca.ModTime()) + } + ca = na +} + +type hashTester struct { + t *testing.T + dir string + env []string +} + +func newHashTester(t *testing.T) *hashTester { + wd, err := os.Getwd() + if err != nil { + fatalf("run failed to get working directory: %v", err) + } + return &hashTester{ + t: t, + dir: wd, + env: os.Environ(), + } +} + +func (h *hashTester) touch(path string) { + path = filepath.Join(h.dir, path) + now := time.Now().UTC() + if err := os.Chtimes(path, now, now); err != nil { + fatalf("failed to touch %v: %v", path, err) + } +} + +func (h *hashTester) statFile(path string) os.FileInfo { + fi, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + fatalf("failed to stat %v: %v", path, err) + } + + return fi +} + +func (h *hashTester) setEnv(key, val string) { + newEnv := []string{fmt.Sprintf("%v=%v", key, val)} + for _, e := range h.env { + if !strings.HasPrefix(e, key+"=") { + newEnv = append(newEnv, e) + } + } + h.env = newEnv +} + +func (h *hashTester) mkdir(dirs ...string) string { + d := filepath.Join(dirs...) + if err := os.MkdirAll(d, 0755); err != nil { + fatalf("failed to mkdir %v: %v\n", d, err) + } + return d +} + +func (h *hashTester) writeFile(path, contents string) { + path = filepath.Join(h.dir, path) + if err := ioutil.WriteFile(path, []byte(contents), 0644); err != nil { + fatalf("failed to write file %v: %v", path, err) + } +} + +func (h *hashTester) tempDir() string { + h.t.Helper() + + td, err := ioutil.TempDir("", "gopherjs_hashTester") + if err != nil { + fatalf("failed to create temp dir: %v", err) + } + + return td +} + +func (h *hashTester) run(c string, args ...string) { + h.t.Helper() + + cmd := exec.Command(c, args...) + cmd.Dir = h.dir + cmd.Env = h.env + + out, err := cmd.CombinedOutput() + if err != nil { + fullCmd := append([]string{c}, args...) + fatalf("failed to run %v: %v\n%v", strings.Join(fullCmd, " "), err, string(out)) + } +} + +func fatalf(format string, args ...interface{}) { + panic(fmt.Errorf(format, args...)) +} diff --git a/tool.go b/tool.go index e3fde81a4..49abc2f92 100644 --- a/tool.go +++ b/tool.go @@ -323,7 +323,6 @@ func main() { cmdTest.Flags().AddFlagSet(compilerFlags) cmdTest.Run = func(cmd *cobra.Command, args []string) { options.BuildTags = strings.Fields(tags) - err := func() error { // Expand import path patterns. patternContext := gbuild.NewBuildContext("", options.BuildTags)