Skip to content

Commit c2773f3

Browse files
grantnelson-wfgrantnelson-wf
committed
Serializing Archieves
1 parent ff99ecc commit c2773f3

File tree

6 files changed

+150
-93
lines changed

6 files changed

+150
-93
lines changed

‎build/build.go

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -977,11 +977,8 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
977977
}
978978

979979
if !s.options.NoCache {
980-
archive := s.buildCache.LoadArchive(pkg.ImportPath, s.Types)
981-
if archive != nil && !pkg.SrcModTime.After(archive.BuildTime) {
982-
if err := archive.RegisterTypes(s.Types); err != nil {
983-
panic(fmt.Errorf("failed to load type information from %v: %w", archive, err))
984-
}
980+
archive := s.buildCache.LoadArchive(pkg.ImportPath, pkg.SrcModTime, s.Types)
981+
if archive != nil {
985982
s.UpToDateArchives[pkg.ImportPath] = archive
986983
// Existing archive is up to date, no need to build it from scratch.
987984
return archive, nil
@@ -1021,7 +1018,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
10211018
fmt.Println(pkg.ImportPath)
10221019
}
10231020

1024-
s.buildCache.StoreArchive(archive)
1021+
s.buildCache.StoreArchive(archive, time.Now())
10251022
s.UpToDateArchives[pkg.ImportPath] = archive
10261023

10271024
return archive, nil

‎build/cache/cache.go

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"os"
1111
"path"
1212
"path/filepath"
13+
"time"
1314

1415
"github.com/gopherjs/gopherjs/compiler"
1516
log "github.com/sirupsen/logrus"
@@ -91,7 +92,7 @@ func (bc BuildCache) String() string {
9192

9293
// StoreArchive compiled archive in the cache. Any error inside this method
9394
// will cause the cache not to be persisted.
94-
func (bc *BuildCache) StoreArchive(a *compiler.Archive) {
95+
func (bc *BuildCache) StoreArchive(a *compiler.Archive, buildTime time.Time) {
9596
if bc == nil {
9697
return // Caching is disabled.
9798
}
@@ -107,7 +108,7 @@ func (bc *BuildCache) StoreArchive(a *compiler.Archive) {
107108
return
108109
}
109110
defer f.Close()
110-
if err := compiler.WriteArchive(a, f); err != nil {
111+
if err := compiler.WriteArchive(a, buildTime, f); err != nil {
111112
log.Warningf("Failed to write build cache archive %q: %v", a, err)
112113
// Make sure we don't leave a half-written archive behind.
113114
os.Remove(f.Name())
@@ -126,7 +127,10 @@ func (bc *BuildCache) StoreArchive(a *compiler.Archive) {
126127
//
127128
// The returned archive would have been built with the same configuration as
128129
// the build cache was.
129-
func (bc *BuildCache) LoadArchive(importPath string, packages map[string]*types.Package) *compiler.Archive {
130+
//
131+
// The imports map is used to resolve package dependencies and may modify the
132+
// map to include the package from the read archive. See [gcexportdata.Read].
133+
func (bc *BuildCache) LoadArchive(importPath string, srcModTime time.Time, imports map[string]*types.Package) *compiler.Archive {
130134
if bc == nil {
131135
return nil // Caching is disabled.
132136
}
@@ -141,12 +145,16 @@ func (bc *BuildCache) LoadArchive(importPath string, packages map[string]*types.
141145
return nil // Cache miss.
142146
}
143147
defer f.Close()
144-
a, err := compiler.ReadArchive(importPath, f, packages)
148+
a, buildTime, err := compiler.ReadArchive(importPath, f, srcModTime, imports)
145149
if err != nil {
146150
log.Warningf("Failed to read cached package archive for %q: %v", importPath, err)
147151
return nil // Invalid/corrupted archive, cache miss.
148152
}
149-
log.Infof("Found cached package archive for %q, built at %v.", importPath, a.BuildTime)
153+
if a == nil {
154+
log.Infof("Found out-of-date package archive for %q, built at %v.", importPath, buildTime)
155+
return nil // Archive is out-of-date, cache miss.
156+
}
157+
log.Infof("Found cached package archive for %q, built at %v.", importPath, buildTime)
150158
return a
151159
}
152160

‎build/cache/cache_test.go

Lines changed: 45 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package cache
33
import (
44
"go/types"
55
"testing"
6+
"time"
67

78
"github.com/google/go-cmp/cmp"
89
"github.com/gopherjs/gopherjs/compiler"
@@ -16,22 +17,24 @@ func TestStore(t *testing.T) {
1617
Imports: []string{"fake/dep"},
1718
}
1819

19-
deps := map[string]*types.Package{}
20+
srcModTime := newTime(0.0)
21+
buildTime := newTime(5.0)
22+
imports := map[string]*types.Package{}
2023
bc := BuildCache{}
21-
if got := bc.LoadArchive(want.ImportPath, deps); got != nil {
24+
if got := bc.LoadArchive(want.ImportPath, srcModTime, imports); got != nil {
2225
t.Errorf("Got: %s was found in the cache. Want: empty cache.", got.ImportPath)
2326
}
24-
bc.StoreArchive(want)
25-
got := bc.LoadArchive(want.ImportPath, deps)
27+
bc.StoreArchive(want, buildTime)
28+
got := bc.LoadArchive(want.ImportPath, srcModTime, imports)
2629
if got == nil {
27-
t.Errorf("Got: %s wan not found in the cache. Want: archive is can be loaded after store.", want.ImportPath)
30+
t.Errorf("Got: %s was not found in the cache. Want: archive is can be loaded after store.", want.ImportPath)
2831
}
2932
if diff := cmp.Diff(want, got); diff != "" {
3033
t.Errorf("Loaded archive is different from stored (-want,+got):\n%s", diff)
3134
}
3235

3336
// Make sure the package names are a part of the cache key.
34-
if got := bc.LoadArchive("fake/other", deps); got != nil {
37+
if got := bc.LoadArchive("fake/other", srcModTime, imports); got != nil {
3538
t.Errorf("Got: fake/other was found in cache: %#v. Want: nil for packages that weren't cached.", got)
3639
}
3740
}
@@ -61,21 +64,54 @@ func TestInvalidation(t *testing.T) {
6164
},
6265
}
6366

64-
deps := map[string]*types.Package{}
67+
srcModTime := newTime(0.0)
68+
buildTime := newTime(5.0)
69+
imports := map[string]*types.Package{}
6570
for _, test := range tests {
6671
a := &compiler.Archive{ImportPath: "package/fake"}
67-
test.cache1.StoreArchive(a)
72+
test.cache1.StoreArchive(a, buildTime)
6873

69-
if got := test.cache2.LoadArchive(a.ImportPath, deps); got != nil {
74+
if got := test.cache2.LoadArchive(a.ImportPath, srcModTime, imports); got != nil {
7075
t.Logf("-cache1,+cache2:\n%s", cmp.Diff(test.cache1, test.cache2))
7176
t.Errorf("Got: %v loaded from cache. Want: build parameter change invalidates cache.", got)
7277
}
7378
}
7479
}
7580

81+
func TestOldArchive(t *testing.T) {
82+
cacheForTest(t)
83+
84+
want := &compiler.Archive{
85+
ImportPath: "fake/package",
86+
Imports: []string{"fake/dep"},
87+
}
88+
89+
buildTime := newTime(5.0)
90+
imports := map[string]*types.Package{}
91+
bc := BuildCache{}
92+
bc.StoreArchive(want, buildTime)
93+
94+
oldSrcModTime := newTime(2.0) // older than archive build time, so archive is up-to-date
95+
got := bc.LoadArchive(want.ImportPath, oldSrcModTime, imports)
96+
if got == nil {
97+
t.Errorf("Got: %s was nil. Want: up-to-date archive to be loaded.", want.ImportPath)
98+
}
99+
100+
newerSrcModTime := newTime(7.0) // newer than archive build time, so archive is stale
101+
got = bc.LoadArchive(want.ImportPath, newerSrcModTime, imports)
102+
if got != nil {
103+
t.Errorf("Got: %s was not nil. Want: stale archive to not be loaded.", want.ImportPath)
104+
}
105+
}
106+
76107
func cacheForTest(t *testing.T) {
77108
t.Helper()
78109
originalRoot := cacheRoot
79110
t.Cleanup(func() { cacheRoot = originalRoot })
80111
cacheRoot = t.TempDir()
81112
}
113+
114+
func newTime(seconds float64) time.Time {
115+
return time.Date(1969, 7, 20, 20, 17, 0, 0, time.UTC).
116+
Add(time.Duration(seconds * float64(time.Second)))
117+
}

‎compiler/compiler.go

Lines changed: 23 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -60,20 +60,12 @@ type Archive struct {
6060
Minified bool
6161
// A list of go:linkname directives encountered in the package.
6262
GoLinknames []GoLinkname
63-
// Time when this archive was built.
64-
BuildTime time.Time
6563
}
6664

6765
func (a Archive) String() string {
6866
return fmt.Sprintf("compiler.Archive{%s}", a.ImportPath)
6967
}
7068

71-
// RegisterTypes adds package type information from the archive into the provided map.
72-
func (a *Archive) RegisterTypes(packages map[string]*types.Package) error {
73-
packages[a.ImportPath] = a.Package
74-
return nil
75-
}
76-
7769
type Dependency struct {
7870
Pkg string
7971
Type string
@@ -277,26 +269,38 @@ type serializableArchive struct {
277269
}
278270

279271
// ReadArchive reads serialized compiled archive of the importPath package.
280-
func ReadArchive(path string, r io.Reader, packages map[string]*types.Package) (*Archive, error) {
272+
//
273+
// The given srcModTime is used to determine if the archive is out-of-date.
274+
// If the archive is out-of-date, the returned archive is nil.
275+
// If there was not an error, the returned time is when the archive was built.
276+
//
277+
// The imports map is used to resolve package dependencies and may modify the
278+
// map to include the package from the read archive. See [gcexportdata.Read].
279+
func ReadArchive(importPath string, r io.Reader, srcModTime time.Time, imports map[string]*types.Package) (*Archive, time.Time, error) {
281280
var sa serializableArchive
282281
if err := gob.NewDecoder(r).Decode(&sa); err != nil {
283-
return nil, err
282+
return nil, time.Time{}, err
283+
}
284+
285+
if srcModTime.After(sa.BuildTime) {
286+
// Archive is out-of-date.
287+
return nil, sa.BuildTime, nil
284288
}
285289

286290
var a Archive
287291
fset := token.NewFileSet()
288292
if len(sa.ExportData) > 0 {
289-
pkg, err := gcexportdata.Read(bytes.NewReader(sa.ExportData), fset, packages, a.ImportPath)
293+
pkg, err := gcexportdata.Read(bytes.NewReader(sa.ExportData), fset, imports, importPath)
290294
if err != nil {
291-
return nil, err
295+
return nil, sa.BuildTime, err
292296
}
293297
a.Package = pkg
294298
}
295299

296300
if len(sa.FileSet) > 0 {
297301
a.FileSet = token.NewFileSet()
298302
if err := a.FileSet.Read(json.NewDecoder(bytes.NewReader(sa.FileSet)).Decode); err != nil {
299-
return nil, err
303+
return nil, sa.BuildTime, err
300304
}
301305
}
302306

@@ -307,12 +311,14 @@ func ReadArchive(path string, r io.Reader, packages map[string]*types.Package) (
307311
a.IncJSCode = sa.IncJSCode
308312
a.Minified = sa.Minified
309313
a.GoLinknames = sa.GoLinknames
310-
a.BuildTime = sa.BuildTime
311-
return &a, nil
314+
return &a, sa.BuildTime, nil
312315
}
313316

314317
// WriteArchive writes compiled package archive on disk for later reuse.
315-
func WriteArchive(a *Archive, w io.Writer) error {
318+
//
319+
// The passed in buildTime is used to determine if the archive is out-of-date.
320+
// It should be set to time.Now() typically but it exposed for testing purposes.
321+
func WriteArchive(a *Archive, buildTime time.Time, w io.Writer) error {
316322
exportData := new(bytes.Buffer)
317323
if a.Package != nil {
318324
if err := gcexportdata.Write(exportData, nil, a.Package); err != nil {
@@ -337,7 +343,7 @@ func WriteArchive(a *Archive, w io.Writer) error {
337343
FileSet: encodedFileSet.Bytes(),
338344
Minified: a.Minified,
339345
GoLinknames: a.GoLinknames,
340-
BuildTime: a.BuildTime,
346+
BuildTime: buildTime,
341347
}
342348

343349
return gob.NewEncoder(w).Encode(sa)

0 commit comments

Comments
 (0)