Skip to content

Archive serialization updates and DCE fix #1354

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Dec 17, 2024
Merged

Archive serialization updates and DCE fix #1354

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
96b85d9
Serialization updates
grantnelson-wf Dec 3, 2024
11bf21d
Using srcModTime instead of time.Now
grantnelson-wf Dec 12, 2024
45e3712
Setting the buildTime back to time.Now
grantnelson-wf Dec 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 35 additions & 21 deletions build/build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"sort"
"strconv"
"strings"
"sync"
"time"

"github.com/fsnotify/fsnotify"
Expand Down Expand Up @@ -938,23 +939,40 @@ func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*Packag
return pkg, archive, nil
}

// getExeModTime will determine the mod time of the GopherJS binary
// the first time this is called and cache the result for subsequent calls.
var getExeModTime = func() func() time.Time {
var (
once sync.Once
result time.Time
)
getTime := func() {
gopherjsBinary, err := os.Executable()
if err == nil {
var fileInfo os.FileInfo
fileInfo, err = os.Stat(gopherjsBinary)
if err == nil {
result = fileInfo.ModTime()
return
}
}
os.Stderr.WriteString("Could not get GopherJS binary's modification timestamp. Please report issue.\n")
result = time.Now()
}
return func() time.Time {
once.Do(getTime)
return result
}
}()

// BuildPackage compiles an already loaded package.
func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
if archive, ok := s.UpToDateArchives[pkg.ImportPath]; ok {
return archive, nil
}

var fileInfo os.FileInfo
gopherjsBinary, err := os.Executable()
if err == nil {
fileInfo, err = os.Stat(gopherjsBinary)
if err == nil && fileInfo.ModTime().After(pkg.SrcModTime) {
pkg.SrcModTime = fileInfo.ModTime()
}
}
if err != nil {
os.Stderr.WriteString("Could not get GopherJS binary's modification timestamp. Please report issue.\n")
pkg.SrcModTime = time.Now()
if exeModTime := getExeModTime(); exeModTime.After(pkg.SrcModTime) {
pkg.SrcModTime = exeModTime
}

for _, importedPkgPath := range pkg.Imports {
Expand All @@ -966,22 +984,18 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
return nil, err
}

impModTime := importedPkg.SrcModTime
if impModTime.After(pkg.SrcModTime) {
if impModTime := importedPkg.SrcModTime; impModTime.After(pkg.SrcModTime) {
pkg.SrcModTime = impModTime
}
}

if pkg.FileModTime().After(pkg.SrcModTime) {
pkg.SrcModTime = pkg.FileModTime()
if fileModTime := pkg.FileModTime(); fileModTime.After(pkg.SrcModTime) {
pkg.SrcModTime = fileModTime
}

if !s.options.NoCache {
archive := s.buildCache.LoadArchive(pkg.ImportPath)
if archive != nil && !pkg.SrcModTime.After(archive.BuildTime) {
if err := archive.RegisterTypes(s.Types); err != nil {
panic(fmt.Errorf("failed to load type information from %v: %w", archive, err))
}
archive := s.buildCache.LoadArchive(pkg.ImportPath, pkg.SrcModTime, s.Types)
if archive != nil {
s.UpToDateArchives[pkg.ImportPath] = archive
// Existing archive is up to date, no need to build it from scratch.
return archive, nil
Expand Down Expand Up @@ -1021,7 +1035,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
fmt.Println(pkg.ImportPath)
}

s.buildCache.StoreArchive(archive)
s.buildCache.StoreArchive(archive, time.Now())
s.UpToDateArchives[pkg.ImportPath] = archive

return archive, nil
Expand Down
22 changes: 17 additions & 5 deletions build/cache/cache.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@ import (
"crypto/sha256"
"fmt"
"go/build"
"go/types"
"os"
"path"
"path/filepath"
"time"

"github.com/gopherjs/gopherjs/compiler"
log "github.com/sirupsen/logrus"
Expand Down Expand Up @@ -90,7 +92,10 @@ func (bc BuildCache) String() string {

// StoreArchive compiled archive in the cache. Any error inside this method
// will cause the cache not to be persisted.
func (bc *BuildCache) StoreArchive(a *compiler.Archive) {
//
// The passed in buildTime is used to determine if the archive is out-of-date when reloaded.
// Typically it should be set to the srcModTime or time.Now().
func (bc *BuildCache) StoreArchive(a *compiler.Archive, buildTime time.Time) {
if bc == nil {
return // Caching is disabled.
}
Expand All @@ -106,7 +111,7 @@ func (bc *BuildCache) StoreArchive(a *compiler.Archive) {
return
}
defer f.Close()
if err := compiler.WriteArchive(a, f); err != nil {
if err := compiler.WriteArchive(a, buildTime, f); err != nil {
log.Warningf("Failed to write build cache archive %q: %v", a, err)
// Make sure we don't leave a half-written archive behind.
os.Remove(f.Name())
Expand All @@ -125,7 +130,10 @@ func (bc *BuildCache) StoreArchive(a *compiler.Archive) {
//
// The returned archive would have been built with the same configuration as
// the build cache was.
func (bc *BuildCache) LoadArchive(importPath string) *compiler.Archive {
//
// The imports map is used to resolve package dependencies and may modify the
// map to include the package from the read archive. See [gcexportdata.Read].
func (bc *BuildCache) LoadArchive(importPath string, srcModTime time.Time, imports map[string]*types.Package) *compiler.Archive {
if bc == nil {
return nil // Caching is disabled.
}
Expand All @@ -140,12 +148,16 @@ func (bc *BuildCache) LoadArchive(importPath string) *compiler.Archive {
return nil // Cache miss.
}
defer f.Close()
a, err := compiler.ReadArchive(importPath, f)
a, buildTime, err := compiler.ReadArchive(importPath, f, srcModTime, imports)
if err != nil {
log.Warningf("Failed to read cached package archive for %q: %v", importPath, err)
return nil // Invalid/corrupted archive, cache miss.
}
log.Infof("Found cached package archive for %q, built at %v.", importPath, a.BuildTime)
if a == nil {
log.Infof("Found out-of-date package archive for %q, built at %v.", importPath, buildTime)
return nil // Archive is out-of-date, cache miss.
}
log.Infof("Found cached package archive for %q, built at %v.", importPath, buildTime)
return a
}

Expand Down
53 changes: 46 additions & 7 deletions build/cache/cache_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
package cache

import (
"go/types"
"testing"
"time"

"github.com/google/go-cmp/cmp"
"github.com/gopherjs/gopherjs/compiler"
Expand All @@ -15,21 +17,24 @@ func TestStore(t *testing.T) {
Imports: []string{"fake/dep"},
}

srcModTime := newTime(0.0)
buildTime := newTime(5.0)
imports := map[string]*types.Package{}
bc := BuildCache{}
if got := bc.LoadArchive(want.ImportPath); got != nil {
if got := bc.LoadArchive(want.ImportPath, srcModTime, imports); got != nil {
t.Errorf("Got: %s was found in the cache. Want: empty cache.", got.ImportPath)
}
bc.StoreArchive(want)
got := bc.LoadArchive(want.ImportPath)
bc.StoreArchive(want, buildTime)
got := bc.LoadArchive(want.ImportPath, srcModTime, imports)
if got == nil {
t.Errorf("Got: %s wan not found in the cache. Want: archive is can be loaded after store.", want.ImportPath)
t.Errorf("Got: %s was not found in the cache. Want: archive is can be loaded after store.", want.ImportPath)
}
if diff := cmp.Diff(want, got); diff != "" {
t.Errorf("Loaded archive is different from stored (-want,+got):\n%s", diff)
}

// Make sure the package names are a part of the cache key.
if got := bc.LoadArchive("fake/other"); got != nil {
if got := bc.LoadArchive("fake/other", srcModTime, imports); got != nil {
t.Errorf("Got: fake/other was found in cache: %#v. Want: nil for packages that weren't cached.", got)
}
}
Expand Down Expand Up @@ -59,20 +64,54 @@ func TestInvalidation(t *testing.T) {
},
}

srcModTime := newTime(0.0)
buildTime := newTime(5.0)
imports := map[string]*types.Package{}
for _, test := range tests {
a := &compiler.Archive{ImportPath: "package/fake"}
test.cache1.StoreArchive(a)
test.cache1.StoreArchive(a, buildTime)

if got := test.cache2.LoadArchive(a.ImportPath); got != nil {
if got := test.cache2.LoadArchive(a.ImportPath, srcModTime, imports); got != nil {
t.Logf("-cache1,+cache2:\n%s", cmp.Diff(test.cache1, test.cache2))
t.Errorf("Got: %v loaded from cache. Want: build parameter change invalidates cache.", got)
}
}
}

func TestOldArchive(t *testing.T) {
cacheForTest(t)

want := &compiler.Archive{
ImportPath: "fake/package",
Imports: []string{"fake/dep"},
}

buildTime := newTime(5.0)
imports := map[string]*types.Package{}
bc := BuildCache{}
bc.StoreArchive(want, buildTime)

oldSrcModTime := newTime(2.0) // older than archive build time, so archive is up-to-date
got := bc.LoadArchive(want.ImportPath, oldSrcModTime, imports)
if got == nil {
t.Errorf("Got: %s was nil. Want: up-to-date archive to be loaded.", want.ImportPath)
}

newerSrcModTime := newTime(7.0) // newer than archive build time, so archive is stale
got = bc.LoadArchive(want.ImportPath, newerSrcModTime, imports)
if got != nil {
t.Errorf("Got: %s was not nil. Want: stale archive to not be loaded.", want.ImportPath)
}
}

func cacheForTest(t *testing.T) {
t.Helper()
originalRoot := cacheRoot
t.Cleanup(func() { cacheRoot = originalRoot })
cacheRoot = t.TempDir()
}

func newTime(seconds float64) time.Time {
return time.Date(1969, 7, 20, 20, 17, 0, 0, time.UTC).
Add(time.Duration(seconds * float64(time.Second)))
}
Loading
Loading