From a0b71356c9063970cb0ff72a54b9fd658e85b15e Mon Sep 17 00:00:00 2001 From: Nevkontakte Date: Sun, 12 Nov 2023 16:30:00 +0000 Subject: [PATCH 01/29] Update Go version to 1.19.13 to begin work on Go 1.19 support. --- .github/workflows/lint.yaml | 2 +- .github/workflows/measure-size.yml | 2 +- README.md | 12 ++++++------ circle.yml | 2 +- compiler/version_check.go | 7 +++---- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 5f6c971e7..03fa75d9c 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -12,7 +12,7 @@ jobs: - uses: actions/setup-go@v3 with: - go-version: "1.18.10" + go-version: "1.19.13" - name: golangci-lint uses: golangci/golangci-lint-action@v3 diff --git a/.github/workflows/measure-size.yml b/.github/workflows/measure-size.yml index d193b0df6..ee4024e6a 100644 --- a/.github/workflows/measure-size.yml +++ b/.github/workflows/measure-size.yml @@ -11,7 +11,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-go@v2 with: - go-version: '~1.18.10' + go-version: '~1.19.13' - uses: gopherjs/output-size-action/measure@main with: name: jQuery TodoMVC diff --git a/README.md b/README.md index 63c5aabee..f29bb9084 100644 --- a/README.md +++ b/README.md @@ -31,21 +31,21 @@ Nearly everything, including Goroutines ([compatibility documentation](https://g ### Installation and Usage -GopherJS [requires Go 1.18 or newer](https://github.com/gopherjs/gopherjs/blob/master/doc/compatibility.md#go-version-compatibility). If you need an older Go +GopherJS [requires Go 1.19 or newer](https://github.com/gopherjs/gopherjs/blob/master/doc/compatibility.md#go-version-compatibility). If you need an older Go version, you can use an [older GopherJS release](https://github.com/gopherjs/gopherjs/releases). Install GopherJS with `go install`: ``` -go install github.com/gopherjs/gopherjs@v1.18.0-beta3 # Or replace 'v1.18.0-beta3' with another version. +go install github.com/gopherjs/gopherjs@v1.19.0-alpha1 # Or replace 'v1.19.0-alpha1' with another version. ``` -If your local Go distribution as reported by `go version` is newer than Go 1.18, then you need to set the `GOPHERJS_GOROOT` environment variable to a directory that contains a Go 1.18 distribution. For example: +If your local Go distribution as reported by `go version` is newer than Go 1.19, then you need to set the `GOPHERJS_GOROOT` environment variable to a directory that contains a Go 1.19 distribution. For example: ``` -go install golang.org/dl/go1.18.10@latest -go1.18.10 download -export GOPHERJS_GOROOT="$(go1.18.10 env GOROOT)" # Also add this line to your .profile or equivalent. +go install golang.org/dl/go1.19.13@latest +go1.19.13 download +export GOPHERJS_GOROOT="$(go1.19.13 env GOROOT)" # Also add this line to your .profile or equivalent. ``` Now you can use `gopherjs build [package]`, `gopherjs build [files]` or `gopherjs install [package]` which behave similar to the `go` tool. For `main` packages, these commands create a `.js` file and `.js.map` source map in the current directory or in `$GOPATH/bin`. The generated JavaScript file can be used as usual in a website. Use `gopherjs help [command]` to get a list of possible command line flags, e.g. for minification and automatically watching for changes. diff --git a/circle.yml b/circle.yml index 12b0aa45f..62031ce4b 100644 --- a/circle.yml +++ b/circle.yml @@ -54,7 +54,7 @@ workflows: parameters: go_version: type: string - default: "1.18.10" + default: "1.19.13" nvm_version: type: string default: "0.38.0" diff --git a/compiler/version_check.go b/compiler/version_check.go index 536bfddca..36bd4acd3 100644 --- a/compiler/version_check.go +++ b/compiler/version_check.go @@ -1,5 +1,4 @@ -//go:build go1.18 -// +build go1.18 +//go:build go1.19 package compiler @@ -13,10 +12,10 @@ import ( ) // Version is the GopherJS compiler version string. -const Version = "1.18.0-beta3+go1.18.10" +const Version = "1.19.0-alpha1+go1.19.13" // GoVersion is the current Go 1.x version that GopherJS is compatible with. -const GoVersion = 18 +const GoVersion = 19 // CheckGoVersion checks the version of the Go distribution // at goroot, and reports an error if it's not compatible From c146af03512be23903ae1e5cb8355242e14f76c8 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 14 Nov 2023 12:20:54 -0700 Subject: [PATCH 02/29] updated reflect and reflectlite --- .../src/internal/reflectlite/reflectlite.go | 29 +++++++++---------- compiler/natives/src/reflect/reflect.go | 23 +++++++-------- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/compiler/natives/src/internal/reflectlite/reflectlite.go b/compiler/natives/src/internal/reflectlite/reflectlite.go index 0be1eb09b..d48f15987 100644 --- a/compiler/natives/src/internal/reflectlite/reflectlite.go +++ b/compiler/natives/src/internal/reflectlite/reflectlite.go @@ -50,7 +50,7 @@ func reflectType(typ *js.Object) *rtype { rt := &rtype{ size: uintptr(typ.Get("size").Int()), kind: uint8(typ.Get("kind").Int()), - str: newNameOff(newName(internalStr(typ.Get("string")), "", typ.Get("exported").Bool())), + str: newNameOff(newName(internalStr(typ.Get("string")), "", typ.Get("exported").Bool(), false)), } js.InternalObject(rt).Set(idJsType, typ) typ.Set(idReflectType, js.InternalObject(rt)) @@ -69,7 +69,7 @@ func reflectType(typ *js.Object) *rtype { continue } reflectMethods = append(reflectMethods, method{ - name: newNameOff(newName(internalStr(m.Get("name")), "", exported)), + name: newNameOff(newName(internalStr(m.Get("name")), "", exported, false)), mtyp: newTypeOff(reflectType(m.Get("typ"))), }) } @@ -81,12 +81,12 @@ func reflectType(typ *js.Object) *rtype { continue } reflectMethods = append(reflectMethods, method{ - name: newNameOff(newName(internalStr(m.Get("name")), "", exported)), + name: newNameOff(newName(internalStr(m.Get("name")), "", exported, false)), mtyp: newTypeOff(reflectType(m.Get("typ"))), }) } ut := &uncommonType{ - pkgPath: newNameOff(newName(internalStr(typ.Get("pkg")), "", false)), + pkgPath: newNameOff(newName(internalStr(typ.Get("pkg")), "", false, false)), mcount: uint16(methodSet.Length()), xcount: xcount, _methods: reflectMethods, @@ -141,13 +141,13 @@ func reflectType(typ *js.Object) *rtype { for i := range imethods { m := methods.Index(i) imethods[i] = imethod{ - name: newNameOff(newName(internalStr(m.Get("name")), "", internalStr(m.Get("pkg")) == "")), + name: newNameOff(newName(internalStr(m.Get("name")), "", internalStr(m.Get("pkg")) == "", false)), typ: newTypeOff(reflectType(m.Get("typ"))), } } setKindType(rt, &interfaceType{ rtype: *rt, - pkgPath: newName(internalStr(typ.Get("pkg")), "", false), + pkgPath: newName(internalStr(typ.Get("pkg")), "", false, false), methods: imethods, }) case Map: @@ -168,19 +168,15 @@ func reflectType(typ *js.Object) *rtype { reflectFields := make([]structField, fields.Length()) for i := range reflectFields { f := fields.Index(i) - offsetEmbed := uintptr(i) << 1 - if f.Get("embedded").Bool() { - offsetEmbed |= 1 - } reflectFields[i] = structField{ - name: newName(internalStr(f.Get("name")), internalStr(f.Get("tag")), f.Get("exported").Bool()), - typ: reflectType(f.Get("typ")), - offsetEmbed: offsetEmbed, + name: newName(internalStr(f.Get("name")), internalStr(f.Get("tag")), f.Get("exported").Bool(), f.Get("embedded").Bool()), + typ: reflectType(f.Get("typ")), + offset: uintptr(i), } } setKindType(rt, &structType{ rtype: *rt, - pkgPath: newName(internalStr(typ.Get("pkgPath")), "", false), + pkgPath: newName(internalStr(typ.Get("pkgPath")), "", false, false), fields: reflectFields, }) } @@ -242,6 +238,7 @@ type nameData struct { name string tag string exported bool + embedded bool } var nameMap = make(map[*byte]*nameData) @@ -250,13 +247,15 @@ func (n name) name() (s string) { return nameMap[n.bytes].name } func (n name) tag() (s string) { return nameMap[n.bytes].tag } func (n name) pkgPath() string { return "" } func (n name) isExported() bool { return nameMap[n.bytes].exported } +func (n name) embedded() bool { return nameMap[n.bytes].embedded } -func newName(n, tag string, exported bool) name { +func newName(n, tag string, exported, embedded bool) name { b := new(byte) nameMap[b] = &nameData{ name: n, tag: tag, exported: exported, + embedded: embedded, } return name{ bytes: b, diff --git a/compiler/natives/src/reflect/reflect.go b/compiler/natives/src/reflect/reflect.go index 3261d70ca..ed5a90835 100644 --- a/compiler/natives/src/reflect/reflect.go +++ b/compiler/natives/src/reflect/reflect.go @@ -63,7 +63,7 @@ func reflectType(typ *js.Object) *rtype { rt := &rtype{ size: uintptr(typ.Get("size").Int()), kind: uint8(typ.Get("kind").Int()), - str: resolveReflectName(newName(internalStr(typ.Get("string")), "", typ.Get("exported").Bool())), + str: resolveReflectName(newName(internalStr(typ.Get("string")), "", typ.Get("exported").Bool(), false)), } js.InternalObject(rt).Set("jsType", typ) typ.Set("reflectType", js.InternalObject(rt)) @@ -99,7 +99,7 @@ func reflectType(typ *js.Object) *rtype { }) } ut := &uncommonType{ - pkgPath: resolveReflectName(newName(internalStr(typ.Get("pkg")), "", false)), + pkgPath: resolveReflectName(newName(internalStr(typ.Get("pkg")), "", false, false)), mcount: uint16(methodSet.Length()), xcount: xcount, _methods: reflectMethods, @@ -160,7 +160,7 @@ func reflectType(typ *js.Object) *rtype { } setKindType(rt, &interfaceType{ rtype: *rt, - pkgPath: newName(internalStr(typ.Get("pkg")), "", false), + pkgPath: newName(internalStr(typ.Get("pkg")), "", false, false), methods: imethods, }) case Map: @@ -181,19 +181,15 @@ func reflectType(typ *js.Object) *rtype { reflectFields := make([]structField, fields.Length()) for i := range reflectFields { f := fields.Index(i) - offsetEmbed := uintptr(i) << 1 - if f.Get("embedded").Bool() { - offsetEmbed |= 1 - } reflectFields[i] = structField{ - name: newName(internalStr(f.Get("name")), internalStr(f.Get("tag")), f.Get("exported").Bool()), - typ: reflectType(f.Get("typ")), - offsetEmbed: offsetEmbed, + name: newName(internalStr(f.Index.Get("name")), internalStr(f.Get("tag")), f.Get("exported").Bool(), f.Get("embedded").Bool()), + typ: reflectType(f.Get("typ")), + offset: uintptr(i), } } setKindType(rt, &structType{ rtype: *rt, - pkgPath: newName(internalStr(typ.Get("pkgPath")), "", false), + pkgPath: newName(internalStr(typ.Get("pkgPath")), "", false, false), fields: reflectFields, }) } @@ -257,6 +253,7 @@ type nameData struct { name string tag string exported bool + embedded bool pkgPath string } @@ -266,16 +263,18 @@ func (n name) name() (s string) { return nameMap[n.bytes].name } func (n name) tag() (s string) { return nameMap[n.bytes].tag } func (n name) pkgPath() string { return nameMap[n.bytes].pkgPath } func (n name) isExported() bool { return nameMap[n.bytes].exported } +func (n name) embedded() bool { return nameMap[n.bytes].embedded } func (n name) setPkgPath(pkgpath string) { nameMap[n.bytes].pkgPath = pkgpath } -func newName(n, tag string, exported bool) name { +func newName(n, tag string, exported, embedded bool) name { b := new(byte) nameMap[b] = &nameData{ name: n, tag: tag, exported: exported, + embedded: embedded, } return name{ bytes: b, From a76a603ce28fd9204ae424987be20333173d7622 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Mon, 4 Dec 2023 13:51:36 -0700 Subject: [PATCH 03/29] Fixing a mistake found in reflect --- compiler/natives/src/reflect/reflect.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/natives/src/reflect/reflect.go b/compiler/natives/src/reflect/reflect.go index ed5a90835..5bcfaf66d 100644 --- a/compiler/natives/src/reflect/reflect.go +++ b/compiler/natives/src/reflect/reflect.go @@ -182,7 +182,7 @@ func reflectType(typ *js.Object) *rtype { for i := range reflectFields { f := fields.Index(i) reflectFields[i] = structField{ - name: newName(internalStr(f.Index.Get("name")), internalStr(f.Get("tag")), f.Get("exported").Bool(), f.Get("embedded").Bool()), + name: newName(internalStr(f.Get("name")), internalStr(f.Get("tag")), f.Get("exported").Bool(), f.Get("embedded").Bool()), typ: reflectType(f.Get("typ")), offset: uintptr(i), } From 5edefc65d084d463ade48366593b198eca38becd Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Mon, 18 Dec 2023 10:27:58 -0700 Subject: [PATCH 04/29] Update FuncKey and fix Http native --- build/build.go | 2 +- compiler/astutil/astutil.go | 33 +++++++++++++++++++---- compiler/astutil/astutil_test.go | 39 +++++++++++++++++++-------- compiler/natives/src/net/http/http.go | 6 ++--- 4 files changed, 60 insertions(+), 20 deletions(-) diff --git a/build/build.go b/build/build.go index 070d05df1..14d2022b7 100644 --- a/build/build.go +++ b/build/build.go @@ -678,7 +678,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { archive := s.buildCache.LoadArchive(pkg.ImportPath) if archive != nil && !pkg.SrcModTime.After(archive.BuildTime) { if err := archive.RegisterTypes(s.Types); err != nil { - panic(fmt.Errorf("Failed to load type information from %v: %w", archive, err)) + panic(fmt.Errorf("failed to load type information from %v: %w", archive, err)) } s.UpToDateArchives[pkg.ImportPath] = archive // Existing archive is up to date, no need to build it from scratch. diff --git a/compiler/astutil/astutil.go b/compiler/astutil/astutil.go index 30febe1cb..b7373f96a 100644 --- a/compiler/astutil/astutil.go +++ b/compiler/astutil/astutil.go @@ -62,14 +62,37 @@ func ImportsUnsafe(file *ast.File) bool { // FuncKey returns a string, which uniquely identifies a top-level function or // method in a package. func FuncKey(d *ast.FuncDecl) string { - if d.Recv == nil || len(d.Recv.List) == 0 { - return d.Name.Name + if recvKey := FuncReceiverKey(d); len(recvKey) > 0 { + return recvKey + "." + d.Name.Name + } + return d.Name.Name +} + +// FuncReceiverKey returns a string that uniquely identifies the receiver +// struct of the function or an empty string if there is no receiver. +// This name will match the name of the struct in the struct's type spec. +func FuncReceiverKey(d *ast.FuncDecl) string { + if d == nil || d.Recv == nil || len(d.Recv.List) == 0 { + return `` } recv := d.Recv.List[0].Type - if star, ok := recv.(*ast.StarExpr); ok { - recv = star.X + for { + switch r := recv.(type) { + case *ast.IndexListExpr: + recv = r.X + continue + case *ast.IndexExpr: + recv = r.X + continue + case *ast.StarExpr: + recv = r.X + continue + case *ast.Ident: + return r.Name + default: + panic(fmt.Errorf(`unexpected type %T in receiver of function: %v`, recv, d)) + } } - return recv.(*ast.Ident).Name + "." + d.Name.Name } // PruneOriginal returns true if gopherjs:prune-original directive is present diff --git a/compiler/astutil/astutil_test.go b/compiler/astutil/astutil_test.go index a996ae73f..2362f6439 100644 --- a/compiler/astutil/astutil_test.go +++ b/compiler/astutil/astutil_test.go @@ -59,24 +59,41 @@ func TestFuncKey(t *testing.T) { want string }{ { - desc: "top-level function", - src: `package testpackage; func foo() {}`, - want: "foo", + desc: `top-level function`, + src: `func foo() {}`, + want: `foo`, + }, { + desc: `top-level exported function`, + src: `func Foo() {}`, + want: `Foo`, + }, { + desc: `method on reference`, + src: `func (_ myType) bar() {}`, + want: `myType.bar`, }, { - desc: "top-level exported function", - src: `package testpackage; func Foo() {}`, - want: "Foo", + desc: `method on pointer`, + src: ` func (_ *myType) bar() {}`, + want: `myType.bar`, }, { - desc: "method", - src: `package testpackage; func (_ myType) bar() {}`, - want: "myType.bar", + desc: `method on generic reference`, + src: ` func (_ myType[T]) bar() {}`, + want: `myType.bar`, + }, { + desc: `method on generic pointer`, + src: ` func (_ *myType[T]) bar() {}`, + want: `myType.bar`, + }, { + desc: `method on struct with multiple generics`, + src: ` func (_ *myType[T1, T2, T3, T4]) bar() {}`, + want: `myType.bar`, }, } for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - fdecl := srctesting.ParseFuncDecl(t, test.src) + src := `package testpackage; ` + test.src + fdecl := srctesting.ParseFuncDecl(t, src) if got := FuncKey(fdecl); got != test.want { - t.Errorf("Got %q, want %q", got, test.want) + t.Errorf(`Got %q, want %q`, got, test.want) } }) } diff --git a/compiler/natives/src/net/http/http.go b/compiler/natives/src/net/http/http.go index 7843235b2..8fd607c4d 100644 --- a/compiler/natives/src/net/http/http.go +++ b/compiler/natives/src/net/http/http.go @@ -7,7 +7,7 @@ import ( "bufio" "bytes" "errors" - "io/ioutil" + "io" "net/textproto" "strconv" @@ -68,7 +68,7 @@ func (t *XHRTransport) RoundTrip(req *Request) (*Response, error) { StatusCode: xhr.Get("status").Int(), Header: Header(header), ContentLength: contentLength, - Body: ioutil.NopCloser(bytes.NewReader(body)), + Body: io.NopCloser(bytes.NewReader(body)), Request: req, } }) @@ -91,7 +91,7 @@ func (t *XHRTransport) RoundTrip(req *Request) (*Response, error) { if req.Body == nil { xhr.Call("send") } else { - body, err := ioutil.ReadAll(req.Body) + body, err := io.ReadAll(req.Body) if err != nil { req.Body.Close() // RoundTrip must always close the body, including on errors. return nil, err From 17263fa5e2ab11d17bdc089852f2f29a0f246fd1 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Mon, 18 Dec 2023 12:16:33 -0700 Subject: [PATCH 05/29] Broke up parseAndAugment --- build/build.go | 257 +++++++++++++++++++++++---------------- build/build_test.go | 290 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 443 insertions(+), 104 deletions(-) diff --git a/build/build.go b/build/build.go index 070d05df1..08c96f4d9 100644 --- a/build/build.go +++ b/build/build.go @@ -117,6 +117,19 @@ func ImportDir(dir string, mode build.ImportMode, installSuffix string, buildTag return pkg, nil } +// overrideInfo is used by parseAndAugment methods to manage +// directives and how the overlay and original are merged. +type overrideInfo struct { + // KeepOriginal indicates that the original code should be kept + // but the identifier will be prefixed by `_gopherjs_original_foo`. + // If false the original code is removed. + keepOriginal bool + + // pruneMethodBody indicates that the body of the methods should be + // removed because they contain something that is invalid to GopherJS. + pruneMethodBody bool +} + // parseAndAugment parses and returns all .go files of given pkg. // Standard Go library packages are augmented with files in compiler/natives folder. // If isTest is true and pkg.ImportPath has no _test suffix, package is built for running internal tests. @@ -132,84 +145,86 @@ func ImportDir(dir string, mode build.ImportMode, installSuffix string, buildTag // the original identifier gets replaced by `_`. New identifiers that don't exist in original // package get added. func parseAndAugment(xctx XContext, pkg *PackageData, isTest bool, fileSet *token.FileSet) ([]*ast.File, []JSFile, error) { - var files []*ast.File + jsFiles, overlayFiles := parseOverlayFiles(xctx, pkg, isTest, fileSet) + + originalFiles, err := parserOriginalFiles(pkg, fileSet) + if err != nil { + return nil, nil, err + } + + overrides := make(map[string]overrideInfo) + for _, file := range overlayFiles { + augmentOverlayFile(file, overrides) + } + delete(overrides, "init") - type overrideInfo struct { - keepOriginal bool - pruneOriginal bool + for _, file := range originalFiles { + augmentOriginalImports(pkg.ImportPath, file) + augmentOriginalFile(file, overrides) } - replacedDeclNames := make(map[string]overrideInfo) + return append(overlayFiles, originalFiles...), jsFiles, nil +} + +// parseOverlayFiles loads and parses overlay files +// to augment the original files with. +func parseOverlayFiles(xctx XContext, pkg *PackageData, isTest bool, fileSet *token.FileSet) ([]JSFile, []*ast.File) { isXTest := strings.HasSuffix(pkg.ImportPath, "_test") importPath := pkg.ImportPath if isXTest { importPath = importPath[:len(importPath)-5] } - jsFiles := []JSFile{} - nativesContext := overlayCtx(xctx.Env()) + nativesPkg, err := nativesContext.Import(importPath, "", 0) + if err != nil { + return nil, nil + } - if nativesPkg, err := nativesContext.Import(importPath, "", 0); err == nil { - jsFiles = nativesPkg.JSFiles - names := nativesPkg.GoFiles - if isTest { - names = append(names, nativesPkg.TestGoFiles...) - } - if isXTest { - names = nativesPkg.XTestGoFiles + jsFiles := nativesPkg.JSFiles + var files []*ast.File + names := nativesPkg.GoFiles + if isTest { + names = append(names, nativesPkg.TestGoFiles...) + } + if isXTest { + names = nativesPkg.XTestGoFiles + } + + for _, name := range names { + fullPath := path.Join(nativesPkg.Dir, name) + r, err := nativesContext.bctx.OpenFile(fullPath) + if err != nil { + panic(err) } - for _, name := range names { - fullPath := path.Join(nativesPkg.Dir, name) - r, err := nativesContext.bctx.OpenFile(fullPath) - if err != nil { - panic(err) - } - // Files should be uniquely named and in the original package directory in order to be - // ordered correctly - newPath := path.Join(pkg.Dir, "gopherjs__"+name) - file, err := parser.ParseFile(fileSet, newPath, r, parser.ParseComments) - if err != nil { - panic(err) - } - r.Close() - for _, decl := range file.Decls { - switch d := decl.(type) { - case *ast.FuncDecl: - k := astutil.FuncKey(d) - replacedDeclNames[k] = overrideInfo{ - keepOriginal: astutil.KeepOriginal(d), - pruneOriginal: astutil.PruneOriginal(d), - } - case *ast.GenDecl: - switch d.Tok { - case token.TYPE: - for _, spec := range d.Specs { - replacedDeclNames[spec.(*ast.TypeSpec).Name.Name] = overrideInfo{} - } - case token.VAR, token.CONST: - for _, spec := range d.Specs { - for _, name := range spec.(*ast.ValueSpec).Names { - replacedDeclNames[name.Name] = overrideInfo{} - } - } - } - } - } - files = append(files, file) + // Files should be uniquely named and in the original package directory in order to be + // ordered correctly + newPath := path.Join(pkg.Dir, "gopherjs__"+name) + file, err := parser.ParseFile(fileSet, newPath, r, parser.ParseComments) + if err != nil { + panic(err) } + r.Close() + + files = append(files, file) } - delete(replacedDeclNames, "init") + return jsFiles, files +} +// parserOriginalFiles loads and parses the original files to augment. +func parserOriginalFiles(pkg *PackageData, fileSet *token.FileSet) ([]*ast.File, error) { + var files []*ast.File var errList compiler.ErrorList for _, name := range pkg.GoFiles { if !filepath.IsAbs(name) { // name might be absolute if specified directly. E.g., `gopherjs build /abs/file.go`. name = filepath.Join(pkg.Dir, name) } + r, err := buildutil.OpenFile(pkg.bctx, name) if err != nil { - return nil, nil, err + return nil, err } + file, err := parser.ParseFile(fileSet, name, r, parser.ParseComments) r.Close() if err != nil { @@ -226,68 +241,102 @@ func parseAndAugment(xctx XContext, pkg *PackageData, isTest bool, fileSet *toke continue } - switch pkg.ImportPath { - case "crypto/rand", "encoding/gob", "encoding/json", "expvar", "go/token", "log", "math/big", "math/rand", "regexp", "time": - for _, spec := range file.Imports { - path, _ := strconv.Unquote(spec.Path.Value) - if path == "sync" { - if spec.Name == nil { - spec.Name = ast.NewIdent("sync") + files = append(files, file) + } + + if errList != nil { + return nil, errList + } + return files, nil +} + +// augmentOverlayFile is the part of parseAndAugment that processes +// an overlay file AST to collect information such as compiler directives +// and perform any initial augmentation needed to the overlay. +func augmentOverlayFile(file *ast.File, overrides map[string]overrideInfo) { + for _, decl := range file.Decls { + switch d := decl.(type) { + case *ast.FuncDecl: + k := astutil.FuncKey(d) + overrides[k] = overrideInfo{ + keepOriginal: astutil.KeepOriginal(d), + pruneMethodBody: astutil.PruneOriginal(d), + } + case *ast.GenDecl: + for _, spec := range d.Specs { + switch s := spec.(type) { + case *ast.TypeSpec: + overrides[s.Name.Name] = overrideInfo{} + case *ast.ValueSpec: + for _, name := range s.Names { + overrides[name.Name] = overrideInfo{} } - spec.Path.Value = `"github.com/gopherjs/gopherjs/nosync"` } } } + } +} - for _, decl := range file.Decls { - switch d := decl.(type) { - case *ast.FuncDecl: - k := astutil.FuncKey(d) - if info, ok := replacedDeclNames[k]; ok { - if info.pruneOriginal { - // Prune function bodies, since it may contain code invalid for - // GopherJS and pin unwanted imports. - d.Body = nil - } - if info.keepOriginal { - // Allow overridden function calls - // The standard library implementation of foo() becomes _gopherjs_original_foo() - d.Name.Name = "_gopherjs_original_" + d.Name.Name - } else { - d.Name = ast.NewIdent("_") - } +// augmentOriginalImports is the part of parseAndAugment that processes +// an original file AST to modify the imports for that file. +func augmentOriginalImports(importPath string, file *ast.File) { + switch importPath { + case "crypto/rand", "encoding/gob", "encoding/json", "expvar", "go/token", "log", "math/big", "math/rand", "regexp", "time": + for _, spec := range file.Imports { + path, _ := strconv.Unquote(spec.Path.Value) + if path == "sync" { + if spec.Name == nil { + spec.Name = ast.NewIdent("sync") } - case *ast.GenDecl: - switch d.Tok { - case token.TYPE: - for _, spec := range d.Specs { - s := spec.(*ast.TypeSpec) - if _, ok := replacedDeclNames[s.Name.Name]; ok { - s.Name = ast.NewIdent("_") - s.Type = &ast.StructType{Struct: s.Pos(), Fields: &ast.FieldList{}} - s.TypeParams = nil - } + spec.Path.Value = `"github.com/gopherjs/gopherjs/nosync"` + } + } + } +} + +// augmentOriginalFile is the part of parseAndAugment that processes an +// original file AST to augment the source code using the overrides from +// the overlay files. +func augmentOriginalFile(file *ast.File, overrides map[string]overrideInfo) { + for _, decl := range file.Decls { + switch d := decl.(type) { + case *ast.FuncDecl: + if info, ok := overrides[astutil.FuncKey(d)]; ok { + if info.pruneMethodBody { + // Prune function bodies, since it may contain code invalid for + // GopherJS and pin unwanted imports. + d.Body = nil + } + if info.keepOriginal { + // Allow overridden function calls + // The standard library implementation of foo() becomes _gopherjs_original_foo() + d.Name.Name = "_gopherjs_original_" + d.Name.Name + } else { + // By setting the name to an underscore, the method will + // not be outputted. Doing this will keep the dependencies the same. + d.Name = ast.NewIdent("_") + } + } + case *ast.GenDecl: + for _, spec := range d.Specs { + switch s := spec.(type) { + case *ast.TypeSpec: + if _, ok := overrides[s.Name.Name]; ok { + s.Name = ast.NewIdent("_") + // Change to struct type with no type body and not type parameters. + s.Type = &ast.StructType{Struct: s.Pos(), Fields: &ast.FieldList{}} + s.TypeParams = nil } - case token.VAR, token.CONST: - for _, spec := range d.Specs { - s := spec.(*ast.ValueSpec) - for i, name := range s.Names { - if _, ok := replacedDeclNames[name.Name]; ok { - s.Names[i] = ast.NewIdent("_") - } + case *ast.ValueSpec: + for i, name := range s.Names { + if _, ok := overrides[name.Name]; ok { + s.Names[i] = ast.NewIdent("_") } } } } } - - files = append(files, file) - } - - if errList != nil { - return nil, nil, errList } - return files, jsFiles, nil } // Options controls build process behavior. diff --git a/build/build_test.go b/build/build_test.go index 2fa17e2c5..8364052d7 100644 --- a/build/build_test.go +++ b/build/build_test.go @@ -1,12 +1,15 @@ package build import ( + "bytes" "fmt" gobuild "go/build" + "go/printer" "go/token" "strconv" "testing" + "github.com/gopherjs/gopherjs/internal/srctesting" "github.com/shurcooL/go/importgraphutil" ) @@ -127,3 +130,290 @@ func (m stringSet) String() string { } return fmt.Sprintf("%q", s) } + +func TestOverlayAugmentation(t *testing.T) { + tests := []struct { + desc string + src string + expInfo map[string]overrideInfo + }{ + { + desc: `remove function`, + src: `func Foo(a, b int) int { + return a + b + }`, + expInfo: map[string]overrideInfo{ + `Foo`: {}, + }, + }, { + desc: `keep function`, + src: `//gopherjs:keep-original + func Foo(a, b int) int { + return a + b + }`, + expInfo: map[string]overrideInfo{ + `Foo`: {keepOriginal: true}, + }, + }, { + desc: `prune function body`, + src: `//gopherjs:prune-original + func Foo(a, b int) int { + return a + b + }`, + expInfo: map[string]overrideInfo{ + `Foo`: {pruneMethodBody: true}, + }, + }, { + desc: `remove constants and values`, + src: `import "time" + + const ( + foo = 42 + bar = "gopherjs" + ) + + var now = time.Now`, + expInfo: map[string]overrideInfo{ + `foo`: {}, + `bar`: {}, + `now`: {}, + }, + }, { + desc: `remove types`, + src: `import "time" + + type ( + foo struct {} + bar int + ) + + type bob interface {}`, + expInfo: map[string]overrideInfo{ + `foo`: {}, + `bar`: {}, + `bob`: {}, + }, + }, { + desc: `remove methods`, + src: `import "cmp" + + type Foo struct { + bar int + } + + func (x *Foo) GetBar() int { return x.bar } + func (x *Foo) SetBar(bar int) { x.bar = bar }`, + expInfo: map[string]overrideInfo{ + `Foo`: {}, + `Foo.GetBar`: {}, + `Foo.SetBar`: {}, + }, + }, { + desc: `remove generics`, + src: `import "cmp" + + type Pointer[T any] struct {} + + func Sort[S ~[]E, E cmp.Ordered](x S) {} + + // this is a stub for "func Equal[S ~[]E, E any](s1, s2 S) bool {}" + func Equal[S ~[]E, E any](s1, s2 S) bool {}`, + expInfo: map[string]overrideInfo{ + `Pointer`: {}, + `Sort`: {}, + `Equal`: {}, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + pkgName := "package testpackage\n\n" + fsetSrc := token.NewFileSet() + fileSrc := srctesting.Parse(t, fsetSrc, pkgName+test.src) + + overrides := map[string]overrideInfo{} + augmentOverlayFile(fileSrc, overrides) + + for key, expInfo := range test.expInfo { + if gotInfo, ok := overrides[key]; !ok { + t.Errorf(`%q was expected but not gotten`, key) + } else if expInfo != gotInfo { + t.Errorf(`%q had wrong info, got %+v`, key, gotInfo) + } + } + for key, gotInfo := range overrides { + if _, ok := test.expInfo[key]; !ok { + t.Errorf(`%q with %+v was not expected`, key, gotInfo) + } + } + }) + } +} + +func TestOriginalAugmentation(t *testing.T) { + tests := []struct { + desc string + info map[string]overrideInfo + src string + want string + }{ + { + desc: `do not affect function`, + info: map[string]overrideInfo{}, + src: `func Foo(a, b int) int { + return a + b + }`, + want: `func Foo(a, b int) int { + return a + b + }`, + }, { + desc: `change unnamed sync import`, + info: map[string]overrideInfo{}, + src: `import "sync" + + var _ = &sync.Mutex{}`, + want: `import sync "github.com/gopherjs/gopherjs/nosync" + + var _ = &sync.Mutex{}`, + }, { + desc: `change named sync import`, + info: map[string]overrideInfo{}, + src: `import foo "sync" + + var _ = &foo.Mutex{}`, + want: `import foo "github.com/gopherjs/gopherjs/nosync" + + var _ = &foo.Mutex{}`, + }, { + desc: `remove function`, + info: map[string]overrideInfo{ + `Foo`: {}, + }, + src: `func Foo(a, b int) int { + return a + b + }`, + want: `func _(a, b int) int { + return a + b + }`, + }, { + desc: `keep original function`, + info: map[string]overrideInfo{ + `Foo`: {keepOriginal: true}, + }, + src: `func Foo(a, b int) int { + return a + b + }`, + want: `func _gopherjs_original_Foo(a, b int) int { + return a + b + }`, + }, { + desc: `remove types and values`, + info: map[string]overrideInfo{ + `Foo`: {}, + `now`: {}, + `bar1`: {}, + }, + src: `import "time" + + type Foo interface{ + bob(a, b string) string + } + + var now = time.Now + const bar1, bar2 = 21, 42`, + want: `import "time" + + type _ struct { + } + + var _ = time.Now + const _, bar2 = 21, 42`, + }, { + desc: `remove in multi-value context`, + info: map[string]overrideInfo{ + `bar`: {}, + }, + src: `const foo, bar = func() (int, int) { + return 24, 12 + }()`, + want: `const foo, _ = func() (int, int) { + return 24, 12 + }()`, + }, { + desc: `remove methods`, + info: map[string]overrideInfo{ + `Foo`: {}, + `Foo.GetBar`: {}, + `Foo.SetBar`: {}, + }, + src: `import "cmp" + + type Foo struct { + bar int + } + + func (x *Foo) GetBar() int { return x.bar } + func (x *Foo) SetBar(bar int) { x.bar = bar }`, + want: `import "cmp" + + type _ struct { + } + + func (x *Foo) _() int { return x.bar } + func (x *Foo) _(bar int) { x.bar = bar }`, + }, { + desc: `remove generics`, + info: map[string]overrideInfo{ + `Pointer`: {}, + `Sort`: {}, + `Equal`: {}, + }, + src: `import "cmp" + + type Pointer[T any] struct {} + + func Sort[S ~[]E, E cmp.Ordered](x S) {} + + // overlay had stub "func Equal() {}" + func Equal[S ~[]E, E any](s1, s2 S) bool {}`, + want: `import "cmp" + + type _ struct { + } + + func _[S ~[]E, E cmp.Ordered](x S) {} + + // overlay had stub "func Equal() {}" + func _[S ~[]E, E any](s1, s2 S) bool {}`, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + pkgName := "package testpackage\n\n" + importPath := `math/rand` + fsetSrc := token.NewFileSet() + fileSrc := srctesting.Parse(t, fsetSrc, pkgName+test.src) + + augmentOriginalImports(importPath, fileSrc) + augmentOriginalFile(fileSrc, test.info) + + buf := &bytes.Buffer{} + _ = printer.Fprint(buf, fsetSrc, fileSrc) + got := buf.String() + + fsetWant := token.NewFileSet() + fileWant := srctesting.Parse(t, fsetWant, pkgName+test.want) + + buf.Reset() + _ = printer.Fprint(buf, fsetWant, fileWant) + want := buf.String() + + if got != want { + t.Errorf("augmentOriginalImports, augmentOriginalFile, and pruneImports got unexpected code:\n"+ + "returned:\n\t%q\nwant:\n\t%q", got, want) + } + }) + } +} From 6e1ec66e5e205d1b581754800b7a1b2251280af4 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Mon, 18 Dec 2023 12:35:48 -0700 Subject: [PATCH 06/29] Extending Directives --- compiler/astutil/astutil.go | 72 ++++-- compiler/astutil/astutil_test.go | 363 ++++++++++++++++++++++++++++++ internal/srctesting/srctesting.go | 36 ++- 3 files changed, 450 insertions(+), 21 deletions(-) diff --git a/compiler/astutil/astutil.go b/compiler/astutil/astutil.go index 30febe1cb..c7481fd27 100644 --- a/compiler/astutil/astutil.go +++ b/compiler/astutil/astutil.go @@ -5,7 +5,7 @@ import ( "go/ast" "go/token" "go/types" - "strings" + "regexp" ) func RemoveParens(e ast.Expr) ast.Expr { @@ -82,15 +82,7 @@ func FuncKey(d *ast.FuncDecl) string { // such as code expecting ints to be 64-bit. It should be used with caution // since it may create unused imports in the original source file. func PruneOriginal(d *ast.FuncDecl) bool { - if d.Doc == nil { - return false - } - for _, c := range d.Doc.List { - if strings.HasPrefix(c.Text, "//gopherjs:prune-original") { - return true - } - } - return false + return hasDirective(d, `prune-original`) } // KeepOriginal returns true if gopherjs:keep-original directive is present @@ -102,15 +94,61 @@ func PruneOriginal(d *ast.FuncDecl) bool { // function in the original called `foo`, it will be accessible by the name // `_gopherjs_original_foo`. func KeepOriginal(d *ast.FuncDecl) bool { - if d.Doc == nil { - return false - } - for _, c := range d.Doc.List { - if strings.HasPrefix(c.Text, "//gopherjs:keep-original") { - return true + return hasDirective(d, `keep-original`) +} + +// anyDocLine calls the given predicate on all associated documentation +// lines and line-comment lines from the given node. +// If the predicate returns true for any line then true is returned. +func anyDocLine(node any, predicate func(line string) bool) bool { + switch a := node.(type) { + case *ast.Comment: + return a != nil && predicate(a.Text) + case *ast.CommentGroup: + if a != nil { + for _, c := range a.List { + if anyDocLine(c, predicate) { + return true + } + } } + return false + case *ast.Field: + return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) + case *ast.File: + return a != nil && anyDocLine(a.Doc, predicate) + case *ast.FuncDecl: + return a != nil && anyDocLine(a.Doc, predicate) + case *ast.GenDecl: + return a != nil && anyDocLine(a.Doc, predicate) + case *ast.ImportSpec: + return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) + case *ast.TypeSpec: + return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) + case *ast.ValueSpec: + return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) + default: + panic(fmt.Errorf(`unexpected node type to get doc from: %T`, node)) } - return false +} + +// directiveMatcher is a regex which matches a GopherJS directive +// and finds the directive action. +var directiveMatcher = regexp.MustCompile(`^\/(?:\/|\*)gopherjs:([\w-]+)`) + +// hasDirective returns true if the associated documentation +// or line comments for the given node have the given directive action. +// +// All GopherJS-specific directives must start with `//gopherjs:` or +// `/*gopherjs:` and followed by an action without any whitespace. The action +// must be one or more letter, decimal, underscore, or hyphen. +// +// see https://pkg.go.dev/cmd/compile#hdr-Compiler_Directives +func hasDirective(node any, directiveAction string) bool { + return anyDocLine(node, func(line string) bool { + m := directiveMatcher.FindStringSubmatch(line) + return len(m) == 2 && m[1] == directiveAction + }) } // FindLoopStmt tries to find the loop statement among the AST nodes in the diff --git a/compiler/astutil/astutil_test.go b/compiler/astutil/astutil_test.go index a996ae73f..61fc2b18c 100644 --- a/compiler/astutil/astutil_test.go +++ b/compiler/astutil/astutil_test.go @@ -1,6 +1,8 @@ package astutil import ( + "fmt" + "go/ast" "go/token" "testing" @@ -130,6 +132,367 @@ func TestPruneOriginal(t *testing.T) { } } +func TestHasDirectiveOnDecl(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: `no comment on function`, + src: `package testpackage; + func foo() {}`, + want: false, + }, { + desc: `no directive on function with comment`, + src: `package testpackage; + // foo has no directive + func foo() {}`, + want: false, + }, { + desc: `wrong directive on function`, + src: `package testpackage; + //gopherjs:wrong-directive + func foo() {}`, + want: false, + }, { + desc: `correct directive on function`, + src: `package testpackage; + //gopherjs:do-stuff + // foo has a directive to do stuff + func foo() {}`, + want: true, + }, { + desc: `correct directive in multiline comment on function`, + src: `package testpackage; + /*gopherjs:do-stuff + foo has a directive to do stuff + */ + func foo() {}`, + want: true, + }, { + desc: `invalid directive in multiline comment on function`, + src: `package testpackage; + /* + gopherjs:do-stuff + */ + func foo() {}`, + want: false, + }, { + desc: `prefix directive on function`, + src: `package testpackage; + //gopherjs:do-stuffs + func foo() {}`, + want: false, + }, { + desc: `multiple directives on function`, + src: `package testpackage; + //gopherjs:wrong-directive + //gopherjs:do-stuff + //gopherjs:another-directive + func foo() {}`, + want: true, + }, { + desc: `directive with explanation on function`, + src: `package testpackage; + //gopherjs:do-stuff 'cause we can + func foo() {}`, + want: true, + }, { + desc: `no directive on type declaration`, + src: `package testpackage; + // Foo has a comment + type Foo int`, + want: false, + }, { + desc: `directive on type declaration`, + src: `package testpackage; + //gopherjs:do-stuff + type Foo int`, + want: true, + }, { + desc: `no directive on const declaration`, + src: `package testpackage; + const foo = 42`, + want: false, + }, { + desc: `directive on const documentation`, + src: `package testpackage; + //gopherjs:do-stuff + const foo = 42`, + want: true, + }, { + desc: `no directive on var declaration`, + src: `package testpackage; + var foo = 42`, + want: false, + }, { + desc: `directive on var documentation`, + src: `package testpackage; + //gopherjs:do-stuff + var foo = 42`, + want: true, + }, { + desc: `no directive on var declaration`, + src: `package testpackage; + import _ "embed"`, + want: false, + }, { + desc: `directive on var documentation`, + src: `package testpackage; + //gopherjs:do-stuff + import _ "embed"`, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + decl := srctesting.ParseDecl(t, test.src) + if got := hasDirective(decl, action); got != test.want { + t.Errorf(`hasDirective(%T, %q) returned %t, want %t`, decl, action, got, test.want) + } + }) + } +} + +func TestHasDirectiveOnSpec(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: `no directive on type specification`, + src: `package testpackage; + type Foo int`, + want: false, + }, { + desc: `directive in doc on type specification`, + src: `package testpackage; + type ( + //gopherjs:do-stuff + Foo int + )`, + want: true, + }, { + desc: `directive in line on type specification`, + src: `package testpackage; + type Foo int //gopherjs:do-stuff`, + want: true, + }, { + desc: `no directive on const specification`, + src: `package testpackage; + const foo = 42`, + want: false, + }, { + desc: `directive in doc on const specification`, + src: `package testpackage; + const ( + //gopherjs:do-stuff + foo = 42 + )`, + want: true, + }, { + desc: `directive in line on const specification`, + src: `package testpackage; + const foo = 42 //gopherjs:do-stuff`, + want: true, + }, { + desc: `no directive on var specification`, + src: `package testpackage; + var foo = 42`, + want: false, + }, { + desc: `directive in doc on var specification`, + src: `package testpackage; + var ( + //gopherjs:do-stuff + foo = 42 + )`, + want: true, + }, { + desc: `directive in line on var specification`, + src: `package testpackage; + var foo = 42 //gopherjs:do-stuff`, + want: true, + }, { + desc: `no directive on import specification`, + src: `package testpackage; + import _ "embed"`, + want: false, + }, { + desc: `directive in doc on import specification`, + src: `package testpackage; + import ( + //gopherjs:do-stuff + _ "embed" + )`, + want: true, + }, { + desc: `directive in line on import specification`, + src: `package testpackage; + import _ "embed" //gopherjs:do-stuff`, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + spec := srctesting.ParseSpec(t, test.src) + if got := hasDirective(spec, action); got != test.want { + t.Errorf(`hasDirective(%T, %q) returned %t, want %t`, spec, action, got, test.want) + } + }) + } +} + +func TestHasDirectiveOnFile(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: `no directive on file`, + src: `package testpackage; + //gopherjs:do-stuff + type Foo int`, + want: false, + }, { + desc: `directive on file`, + src: `//gopherjs:do-stuff + package testpackage; + type Foo int`, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + fset := token.NewFileSet() + file := srctesting.Parse(t, fset, test.src) + if got := hasDirective(file, action); got != test.want { + t.Errorf(`hasDirective(%T, %q) returned %t, want %t`, file, action, got, test.want) + } + }) + } +} + +func TestHasDirectiveOnField(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: `no directive on struct field`, + src: `package testpackage; + type Foo struct { + bar int + }`, + want: false, + }, { + desc: `directive in doc on struct field`, + src: `package testpackage; + type Foo struct { + //gopherjs:do-stuff + bar int + }`, + want: true, + }, { + desc: `directive in line on struct field`, + src: `package testpackage; + type Foo struct { + bar int //gopherjs:do-stuff + }`, + want: true, + }, { + desc: `no directive on interface method`, + src: `package testpackage; + type Foo interface { + Bar(a int) int + }`, + want: false, + }, { + desc: `directive in doc on interface method`, + src: `package testpackage; + type Foo interface { + //gopherjs:do-stuff + Bar(a int) int + }`, + want: true, + }, { + desc: `directive in line on interface method`, + src: `package testpackage; + type Foo interface { + Bar(a int) int //gopherjs:do-stuff + }`, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + spec := srctesting.ParseSpec(t, test.src) + tspec := spec.(*ast.TypeSpec) + var field *ast.Field + switch typeNode := tspec.Type.(type) { + case *ast.StructType: + field = typeNode.Fields.List[0] + case *ast.InterfaceType: + field = typeNode.Methods.List[0] + default: + t.Errorf(`unexpected node type, %T, when finding field`, typeNode) + return + } + if got := hasDirective(field, action); got != test.want { + t.Errorf(`hasDirective(%T, %q) returned %t, want %t`, field, action, got, test.want) + } + }) + } +} + +func TestHasDirectiveBadCase(t *testing.T) { + tests := []struct { + desc string + node any + want string + }{ + { + desc: `untyped nil node`, + node: nil, + want: `unexpected node type to get doc from: `, + }, { + desc: `unexpected node type`, + node: &ast.ArrayType{}, + want: `unexpected node type to get doc from: *ast.ArrayType`, + }, { + desc: `nil expected node type`, + node: (*ast.FuncDecl)(nil), + want: ``, // no panic + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + var got string + func() { + defer func() { got = fmt.Sprint(recover()) }() + hasDirective(test.node, action) + }() + if got != test.want { + t.Errorf(`hasDirective(%T, %q) returned %s, want %s`, test.node, action, got, test.want) + } + }) + } +} + func TestEndsWithReturn(t *testing.T) { tests := []struct { desc string diff --git a/internal/srctesting/srctesting.go b/internal/srctesting/srctesting.go index 1d9cecd20..4e374845e 100644 --- a/internal/srctesting/srctesting.go +++ b/internal/srctesting/srctesting.go @@ -53,17 +53,45 @@ func Check(t *testing.T, fset *token.FileSet, files ...*ast.File) (*types.Info, // // Fails the test if there isn't exactly one function declared in the source. func ParseFuncDecl(t *testing.T, src string) *ast.FuncDecl { + t.Helper() + decl := ParseDecl(t, src) + fdecl, ok := decl.(*ast.FuncDecl) + if !ok { + t.Fatalf("Got %T decl, expected *ast.FuncDecl", decl) + } + return fdecl +} + +// ParseDecl parses source with a single declaration and +// returns that declaration AST. +// +// Fails the test if there isn't exactly one declaration in the source. +func ParseDecl(t *testing.T, src string) ast.Decl { t.Helper() fset := token.NewFileSet() file := Parse(t, fset, src) if l := len(file.Decls); l != 1 { - t.Fatalf("Got %d decls in the sources, expected exactly 1", l) + t.Fatalf(`Got %d decls in the sources, expected exactly 1`, l) } - fdecl, ok := file.Decls[0].(*ast.FuncDecl) + return file.Decls[0] +} + +// ParseSpec parses source with a single declaration containing +// a single specification and returns that specification AST. +// +// Fails the test if there isn't exactly one declaration and +// one specification in the source. +func ParseSpec(t *testing.T, src string) ast.Spec { + t.Helper() + decl := ParseDecl(t, src) + gdecl, ok := decl.(*ast.GenDecl) if !ok { - t.Fatalf("Got %T decl, expected *ast.FuncDecl", file.Decls[0]) + t.Fatalf("Got %T decl, expected *ast.GenDecl", decl) } - return fdecl + if l := len(gdecl.Specs); l != 1 { + t.Fatalf(`Got %d spec in the sources, expected exactly 1`, l) + } + return gdecl.Specs[0] } // Format AST node into a string. From 27e12978b636af57adbdab76ba3e739f72a60b44 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Mon, 18 Dec 2023 16:25:37 -0700 Subject: [PATCH 07/29] Adding purge directive --- build/build.go | 214 ++++++++++++++++++---- build/build_test.go | 292 +++++++++++++++++++++++++------ compiler/astutil/astutil.go | 88 ++++++++-- compiler/astutil/astutil_test.go | 148 +++++++++++----- 4 files changed, 600 insertions(+), 142 deletions(-) diff --git a/build/build.go b/build/build.go index 4cbaa5f32..b32ed9f37 100644 --- a/build/build.go +++ b/build/build.go @@ -125,9 +125,11 @@ type overrideInfo struct { // If false the original code is removed. keepOriginal bool - // pruneMethodBody indicates that the body of the methods should be - // removed because they contain something that is invalid to GopherJS. - pruneMethodBody bool + // purgeMethods indicates that this info is for a type and + // if a method has this type as a receiver should also be removed. + // If the method is defined in the overlays and therefore has its + // own overrides, this will be ignored. + purgeMethods bool } // parseAndAugment parses and returns all .go files of given pkg. @@ -138,12 +140,19 @@ type overrideInfo struct { // The native packages are augmented by the contents of natives.FS in the following way. // The file names do not matter except the usual `_test` suffix. The files for // native overrides get added to the package (even if they have the same name -// as an existing file from the standard library). For function identifiers that exist -// in the original AND the overrides AND that include the following directive in their comment: -// //gopherjs:keep-original, the original identifier in the AST gets prefixed by -// `_gopherjs_original_`. For other identifiers that exist in the original AND the overrides, -// the original identifier gets replaced by `_`. New identifiers that don't exist in original -// package get added. +// as an existing file from the standard library). +// +// - For function identifiers that exist in the original and the overrides +// and have the directive `gopherjs:keep-original`, the original identifier +// in the AST gets prefixed by `_gopherjs_original_`. +// - For identifiers that exist in the original and the overrides and have +// the directive `gopherjs:purge`, both the original and override are +// removed. This is for completely removing something which is currently +// invalid for GopherJS. For any purged types any methods with that type as +// the receiver are also removed. +// - Otherwise for identifiers that exist in the original and the overrides, +// the original is removed. +// - New identifiers that don't exist in original package get added. func parseAndAugment(xctx XContext, pkg *PackageData, isTest bool, fileSet *token.FileSet) ([]*ast.File, []JSFile, error) { jsFiles, overlayFiles := parseOverlayFiles(xctx, pkg, isTest, fileSet) @@ -155,12 +164,14 @@ func parseAndAugment(xctx XContext, pkg *PackageData, isTest bool, fileSet *toke overrides := make(map[string]overrideInfo) for _, file := range overlayFiles { augmentOverlayFile(file, overrides) + pruneImports(file) } delete(overrides, "init") for _, file := range originalFiles { augmentOriginalImports(pkg.ImportPath, file) augmentOriginalFile(file, overrides) + pruneImports(file) } return append(overlayFiles, originalFiles...), jsFiles, nil @@ -254,27 +265,37 @@ func parserOriginalFiles(pkg *PackageData, fileSet *token.FileSet) ([]*ast.File, // an overlay file AST to collect information such as compiler directives // and perform any initial augmentation needed to the overlay. func augmentOverlayFile(file *ast.File, overrides map[string]overrideInfo) { - for _, decl := range file.Decls { + for i, decl := range file.Decls { + purgeDecl := astutil.Purge(decl) switch d := decl.(type) { case *ast.FuncDecl: k := astutil.FuncKey(d) overrides[k] = overrideInfo{ - keepOriginal: astutil.KeepOriginal(d), - pruneMethodBody: astutil.PruneOriginal(d), + keepOriginal: astutil.KeepOriginal(d), } case *ast.GenDecl: - for _, spec := range d.Specs { + for j, spec := range d.Specs { + purgeSpec := purgeDecl || astutil.Purge(spec) switch s := spec.(type) { case *ast.TypeSpec: - overrides[s.Name.Name] = overrideInfo{} + overrides[s.Name.Name] = overrideInfo{ + purgeMethods: purgeSpec, + } case *ast.ValueSpec: for _, name := range s.Names { overrides[name.Name] = overrideInfo{} } } + if purgeSpec { + d.Specs[j] = nil + } } } + if purgeDecl { + file.Decls[i] = nil + } } + finalizeRemovals(file) } // augmentOriginalImports is the part of parseAndAugment that processes @@ -298,45 +319,176 @@ func augmentOriginalImports(importPath string, file *ast.File) { // original file AST to augment the source code using the overrides from // the overlay files. func augmentOriginalFile(file *ast.File, overrides map[string]overrideInfo) { - for _, decl := range file.Decls { + for i, decl := range file.Decls { switch d := decl.(type) { case *ast.FuncDecl: if info, ok := overrides[astutil.FuncKey(d)]; ok { - if info.pruneMethodBody { - // Prune function bodies, since it may contain code invalid for - // GopherJS and pin unwanted imports. - d.Body = nil - } if info.keepOriginal { // Allow overridden function calls // The standard library implementation of foo() becomes _gopherjs_original_foo() d.Name.Name = "_gopherjs_original_" + d.Name.Name } else { - // By setting the name to an underscore, the method will - // not be outputted. Doing this will keep the dependencies the same. - d.Name = ast.NewIdent("_") + file.Decls[i] = nil + } + } else if recvKey := astutil.FuncReceiverKey(d); len(recvKey) > 0 { + // check if the receiver has been purged, if so, remove the method too. + if info, ok := overrides[recvKey]; ok && info.purgeMethods { + file.Decls[i] = nil } } case *ast.GenDecl: - for _, spec := range d.Specs { + for j, spec := range d.Specs { switch s := spec.(type) { case *ast.TypeSpec: if _, ok := overrides[s.Name.Name]; ok { - s.Name = ast.NewIdent("_") - // Change to struct type with no type body and not type parameters. - s.Type = &ast.StructType{Struct: s.Pos(), Fields: &ast.FieldList{}} - s.TypeParams = nil + d.Specs[j] = nil } case *ast.ValueSpec: - for i, name := range s.Names { - if _, ok := overrides[name.Name]; ok { - s.Names[i] = ast.NewIdent("_") + if len(s.Names) == len(s.Values) { + // multi-value context + // e.g. var a, b = 2, foo[int]() + // A removal will also remove the value which may be from a + // function call. This allows us to remove unwanted statements. + // However, if that call has a side effect which still needs + // to be run, add the call into the overlay. + for k, name := range s.Names { + if _, ok := overrides[name.Name]; ok { + s.Names[k] = nil + s.Values[k] = nil + } + } + } else { + // single-value context + // e.g. var a, b = foo[int]() + // If a removal from the overlays makes all returned values unused, + // then remove the function call as well. This allows us to stop + // unwanted calls if needed. If that call has a side effect which + // still needs to be run, add the call into the overlay. + nameRemoved := false + for _, name := range s.Names { + if _, ok := overrides[name.Name]; ok { + nameRemoved = true + name.Name = `_` + } + } + if nameRemoved { + removeSpec := true + for _, name := range s.Names { + if name.Name != `_` { + removeSpec = false + break + } + } + if removeSpec { + d.Specs[j] = nil + } } } } } } } + finalizeRemovals(file) +} + +// pruneImports will remove any unused imports from the file. +// +// This will not remove any dot (`.`) or blank (`_`) imports. +// If the removal of code causes an import to be removed, the init's from that +// import may not be run anymore. If we still need to run an init for an import +// which is no longer used, add it to the overlay as a blank (`_`) import. +func pruneImports(file *ast.File) { + unused := make(map[string]int, len(file.Imports)) + for i, in := range file.Imports { + if name := astutil.ImportName(in); len(name) > 0 { + unused[name] = i + } + } + + // Remove "unused import" for any import which is used. + ast.Walk(astutil.NewCallbackVisitor(func(n ast.Node) bool { + if sel, ok := n.(*ast.SelectorExpr); ok { + if id, ok := sel.X.(*ast.Ident); ok && id.Obj == nil { + delete(unused, id.Name) + } + } + return len(unused) > 0 + }), file) + if len(unused) == 0 { + return + } + + // Remove all unused import specifications + isUnusedSpec := map[*ast.ImportSpec]bool{} + for _, index := range unused { + isUnusedSpec[file.Imports[index]] = true + } + for _, decl := range file.Decls { + if d, ok := decl.(*ast.GenDecl); ok { + for i, spec := range d.Specs { + if other, ok := spec.(*ast.ImportSpec); ok && isUnusedSpec[other] { + d.Specs[i] = nil + } + } + } + } + + // Remove the unused import copies in the file + for _, index := range unused { + file.Imports[index] = nil + } + + finalizeRemovals(file) +} + +// finalizeRemovals fully removes any declaration, specification, imports +// that have been set to nil. This will also remove the file's top-level +// comment group to remove any unassociated comments, including the comments +// from removed code. +func finalizeRemovals(file *ast.File) { + fileChanged := false + for i, decl := range file.Decls { + switch d := decl.(type) { + case nil: + fileChanged = true + case *ast.GenDecl: + declChanged := false + for j, spec := range d.Specs { + switch s := spec.(type) { + case nil: + declChanged = true + case *ast.ValueSpec: + specChanged := false + for _, name := range s.Names { + if name == nil { + specChanged = true + break + } + } + if specChanged { + s.Names = astutil.Squeeze(s.Names) + s.Values = astutil.Squeeze(s.Values) + if len(s.Names) == 0 { + declChanged = true + d.Specs[j] = nil + } + } + } + } + if declChanged { + d.Specs = astutil.Squeeze(d.Specs) + if len(d.Specs) == 0 { + fileChanged = true + file.Decls[i] = nil + } + } + } + } + if fileChanged { + file.Decls = astutil.Squeeze(file.Decls) + } + file.Imports = astutil.Squeeze(file.Imports) + file.Comments = nil } // Options controls build process behavior. diff --git a/build/build_test.go b/build/build_test.go index 8364052d7..81a12b36d 100644 --- a/build/build_test.go +++ b/build/build_test.go @@ -133,15 +133,18 @@ func (m stringSet) String() string { func TestOverlayAugmentation(t *testing.T) { tests := []struct { - desc string - src string - expInfo map[string]overrideInfo + desc string + src string + noCodeChange bool + want string + expInfo map[string]overrideInfo }{ { desc: `remove function`, src: `func Foo(a, b int) int { - return a + b - }`, + return a + b + }`, + noCodeChange: true, expInfo: map[string]overrideInfo{ `Foo`: {}, }, @@ -151,18 +154,10 @@ func TestOverlayAugmentation(t *testing.T) { func Foo(a, b int) int { return a + b }`, + noCodeChange: true, expInfo: map[string]overrideInfo{ `Foo`: {keepOriginal: true}, }, - }, { - desc: `prune function body`, - src: `//gopherjs:prune-original - func Foo(a, b int) int { - return a + b - }`, - expInfo: map[string]overrideInfo{ - `Foo`: {pruneMethodBody: true}, - }, }, { desc: `remove constants and values`, src: `import "time" @@ -173,6 +168,7 @@ func TestOverlayAugmentation(t *testing.T) { ) var now = time.Now`, + noCodeChange: true, expInfo: map[string]overrideInfo{ `foo`: {}, `bar`: {}, @@ -180,14 +176,13 @@ func TestOverlayAugmentation(t *testing.T) { }, }, { desc: `remove types`, - src: `import "time" - - type ( + src: `type ( foo struct {} bar int ) type bob interface {}`, + noCodeChange: true, expInfo: map[string]overrideInfo{ `foo`: {}, `bar`: {}, @@ -195,14 +190,13 @@ func TestOverlayAugmentation(t *testing.T) { }, }, { desc: `remove methods`, - src: `import "cmp" - - type Foo struct { + src: `type Foo struct { bar int } - + func (x *Foo) GetBar() int { return x.bar } func (x *Foo) SetBar(bar int) { x.bar = bar }`, + noCodeChange: true, expInfo: map[string]overrideInfo{ `Foo`: {}, `Foo.GetBar`: {}, @@ -218,22 +212,199 @@ func TestOverlayAugmentation(t *testing.T) { // this is a stub for "func Equal[S ~[]E, E any](s1, s2 S) bool {}" func Equal[S ~[]E, E any](s1, s2 S) bool {}`, + noCodeChange: true, expInfo: map[string]overrideInfo{ `Pointer`: {}, `Sort`: {}, `Equal`: {}, }, + }, { + desc: `prune an unused import`, + src: `import foo "some/other/bar"`, + want: ``, + expInfo: map[string]overrideInfo{}, + }, { + desc: `purge function`, + src: `//gopherjs:purge + func Foo(a, b int) int { + return a + b + }`, + want: ``, + expInfo: map[string]overrideInfo{ + `Foo`: {}, + }, + }, { + desc: `purge struct removes an import`, + src: `import "bytes" + import "math" + + //gopherjs:purge + type Foo struct { + bar *bytes.Buffer + } + + const Tau = math.Pi * 2.0`, + want: `import "math" + + const Tau = math.Pi * 2.0`, + expInfo: map[string]overrideInfo{ + `Foo`: {purgeMethods: true}, + `Tau`: {}, + }, + }, { + desc: `purge whole type decl`, + src: `//gopherjs:purge + type ( + Foo struct {} + bar interface{} + bob int + )`, + want: ``, + expInfo: map[string]overrideInfo{ + `Foo`: {purgeMethods: true}, + `bar`: {purgeMethods: true}, + `bob`: {purgeMethods: true}, + }, + }, { + desc: `purge part of type decl`, + src: `type ( + Foo struct {} + + //gopherjs:purge + bar interface{} + + //gopherjs:purge + bob int + )`, + want: `type ( + Foo struct {} + )`, + expInfo: map[string]overrideInfo{ + `Foo`: {}, + `bar`: {purgeMethods: true}, + `bob`: {purgeMethods: true}, + }, + }, { + desc: `purge all of a type decl`, + src: `type ( + //gopherjs:purge + Foo struct {} + )`, + want: ``, + expInfo: map[string]overrideInfo{ + `Foo`: {purgeMethods: true}, + }, + }, { + desc: `remove and purge values`, + src: `import "time" + + const ( + foo = 42 + //gopherjs:purge + bar = "gopherjs" + ) + + //gopherjs:purge + var now = time.Now`, + want: `const ( + foo = 42 + )`, + expInfo: map[string]overrideInfo{ + `foo`: {}, + `bar`: {}, + `now`: {}, + }, + }, { + desc: `purge all value names`, + src: `//gopherjs:purge + var foo, bar int + + //gopherjs:purge + const bob, sal = 12, 42`, + want: ``, + expInfo: map[string]overrideInfo{ + `foo`: {}, + `bar`: {}, + `bob`: {}, + `sal`: {}, + }, + }, { + desc: `imports not confused by local variables`, + src: `import ( + "cmp" + "time" + ) + + //gopherjs:purge + func Sort[S ~[]E, E cmp.Ordered](x S) {} + + func SecondsSince(start time.Time) int { + cmp := time.Now().Sub(start) + return int(cmp.Second()) + }`, + want: `import ( + "time" + ) + + func SecondsSince(start time.Time) int { + cmp := time.Now().Sub(start) + return int(cmp.Second()) + }`, + expInfo: map[string]overrideInfo{ + `Sort`: {}, + `SecondsSince`: {}, + }, + }, { + desc: `purge generics`, + src: `import "cmp" + + //gopherjs:purge + type Pointer[T any] struct {} + + //gopherjs:purge + func Sort[S ~[]E, E cmp.Ordered](x S) {} + + // stub for "func Equal[S ~[]E, E any](s1, s2 S) bool" + func Equal() {}`, + want: `// stub for "func Equal[S ~[]E, E any](s1, s2 S) bool" + func Equal() {}`, + expInfo: map[string]overrideInfo{ + `Pointer`: {purgeMethods: true}, + `Sort`: {}, + `Equal`: {}, + }, }, } for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - pkgName := "package testpackage\n\n" + const pkgName = "package testpackage\n\n" + if test.noCodeChange { + test.want = test.src + } + fsetSrc := token.NewFileSet() fileSrc := srctesting.Parse(t, fsetSrc, pkgName+test.src) overrides := map[string]overrideInfo{} augmentOverlayFile(fileSrc, overrides) + pruneImports(fileSrc) + + buf := &bytes.Buffer{} + _ = printer.Fprint(buf, fsetSrc, fileSrc) + got := buf.String() + + fsetWant := token.NewFileSet() + fileWant := srctesting.Parse(t, fsetWant, pkgName+test.want) + + buf.Reset() + _ = printer.Fprint(buf, fsetWant, fileWant) + want := buf.String() + + if got != want { + t.Errorf("augmentOverlayFile and pruneImports got unexpected code:\n"+ + "returned:\n\t%q\nwant:\n\t%q", got, want) + } for key, expInfo := range test.expInfo { if gotInfo, ok := overrides[key]; !ok { @@ -293,9 +464,7 @@ func TestOriginalAugmentation(t *testing.T) { src: `func Foo(a, b int) int { return a + b }`, - want: `func _(a, b int) int { - return a + b - }`, + want: ``, }, { desc: `keep original function`, info: map[string]overrideInfo{ @@ -322,13 +491,7 @@ func TestOriginalAugmentation(t *testing.T) { var now = time.Now const bar1, bar2 = 21, 42`, - want: `import "time" - - type _ struct { - } - - var _ = time.Now - const _, bar2 = 21, 42`, + want: `const bar2 = 42`, }, { desc: `remove in multi-value context`, info: map[string]overrideInfo{ @@ -340,28 +503,41 @@ func TestOriginalAugmentation(t *testing.T) { want: `const foo, _ = func() (int, int) { return 24, 12 }()`, + }, { + desc: `full remove in multi-value context`, + info: map[string]overrideInfo{ + `bar`: {}, + }, + src: `const _, bar = func() (int, int) { + return 24, 12 + }()`, + want: ``, }, { desc: `remove methods`, info: map[string]overrideInfo{ - `Foo`: {}, `Foo.GetBar`: {}, `Foo.SetBar`: {}, }, - src: `import "cmp" - - type Foo struct { + src: ` + func (x Foo) GetBar() int { return x.bar } + func (x *Foo) SetBar(bar int) { x.bar = bar }`, + want: ``, + }, { + desc: `purge struct and methods`, + info: map[string]overrideInfo{ + `Foo`: {purgeMethods: true}, + }, + src: `type Foo struct{ bar int } - - func (x *Foo) GetBar() int { return x.bar } - func (x *Foo) SetBar(bar int) { x.bar = bar }`, - want: `import "cmp" - type _ struct { - } - - func (x *Foo) _() int { return x.bar } - func (x *Foo) _(bar int) { x.bar = bar }`, + func (f Foo) GetBar() int { return f.bar } + func (f *Foo) SetBar(bar int) { f.bar = bar } + + func NewFoo(bar int) *Foo { return &Foo{bar: bar} }`, + // NewFoo is not removed automatically since + // only functions with Foo as a receiver is removed. + want: `func NewFoo(bar int) *Foo { return &Foo{bar: bar} }`, }, { desc: `remove generics`, info: map[string]overrideInfo{ @@ -377,15 +553,30 @@ func TestOriginalAugmentation(t *testing.T) { // overlay had stub "func Equal() {}" func Equal[S ~[]E, E any](s1, s2 S) bool {}`, - want: `import "cmp" + want: ``, + }, { + desc: `purge generics`, + info: map[string]overrideInfo{ + `Pointer`: {purgeMethods: true}, + `Sort`: {}, + `Equal`: {}, + }, + src: `import "cmp" - type _ struct { - } + type Pointer[T any] struct {} + func (x *Pointer[T]) Load() *T {} + func (x *Pointer[T]) Store(val *T) {} - func _[S ~[]E, E cmp.Ordered](x S) {} + func Sort[S ~[]E, E cmp.Ordered](x S) {} // overlay had stub "func Equal() {}" - func _[S ~[]E, E any](s1, s2 S) bool {}`, + func Equal[S ~[]E, E any](s1, s2 S) bool {}`, + want: ``, + }, { + desc: `prune an unused import`, + info: map[string]overrideInfo{}, + src: `import foo "some/other/bar"`, + want: ``, }, } @@ -398,6 +589,7 @@ func TestOriginalAugmentation(t *testing.T) { augmentOriginalImports(importPath, fileSrc) augmentOriginalFile(fileSrc, test.info) + pruneImports(fileSrc) buf := &bytes.Buffer{} _ = printer.Fprint(buf, fsetSrc, fileSrc) diff --git a/compiler/astutil/astutil.go b/compiler/astutil/astutil.go index 79292003d..24c1803c0 100644 --- a/compiler/astutil/astutil.go +++ b/compiler/astutil/astutil.go @@ -5,7 +5,10 @@ import ( "go/ast" "go/token" "go/types" + "path" + "reflect" "regexp" + "strconv" ) func RemoveParens(e ast.Expr) ast.Expr { @@ -59,6 +62,29 @@ func ImportsUnsafe(file *ast.File) bool { return false } +// ImportName tries to determine the package name for an import. +// +// If the package name isn't specified then this will make a best +// make a best guess using the import path. +// If the import name is dot (`.`), blank (`_`), or there +// was an issue determining the package name then empty is returned. +func ImportName(spec *ast.ImportSpec) string { + var name string + if spec.Name != nil { + name = spec.Name.Name + } else { + importPath, _ := strconv.Unquote(spec.Path.Value) + name = path.Base(importPath) + } + + switch name { + case `_`, `.`, `/`: + return `` + default: + return name + } +} + // FuncKey returns a string, which uniquely identifies a top-level function or // method in a package. func FuncKey(d *ast.FuncDecl) string { @@ -95,19 +121,6 @@ func FuncReceiverKey(d *ast.FuncDecl) string { } } -// PruneOriginal returns true if gopherjs:prune-original directive is present -// before a function decl. -// -// `//gopherjs:prune-original` is a GopherJS-specific directive, which can be -// applied to functions in native overlays and will instruct the augmentation -// logic to delete the body of a standard library function that was replaced. -// This directive can be used to remove code that would be invalid in GopherJS, -// such as code expecting ints to be 64-bit. It should be used with caution -// since it may create unused imports in the original source file. -func PruneOriginal(d *ast.FuncDecl) bool { - return hasDirective(d, `prune-original`) -} - // KeepOriginal returns true if gopherjs:keep-original directive is present // before a function decl. // @@ -120,6 +133,21 @@ func KeepOriginal(d *ast.FuncDecl) bool { return hasDirective(d, `keep-original`) } +// Purge returns true if gopherjs:purge directive is present +// on a struct, interface, type, variable, constant, or function. +// +// `//gopherjs:purge` is a GopherJS-specific directive, which can be +// applied in native overlays and will instruct the augmentation logic to +// delete part of the standard library without a replacement. This directive +// can be used to remove code that would be invalid in GopherJS, such as code +// using unsupported features (e.g. generic interfaces before generics were +// fully supported). It should be used with caution since it may remove needed +// dependencies. If a type is purged, all methods using that type as +// a receiver will also be purged. +func Purge(d any) bool { + return hasDirective(d, `purge`) +} + // anyDocLine calls the given predicate on all associated documentation // lines and line-comment lines from the given node. // If the predicate returns true for any line then true is returned. @@ -228,3 +256,37 @@ func EndsWithReturn(stmts []ast.Stmt) bool { return false } } + +// Squeeze removes all nil nodes from the slice. +// +// The given slice will be modified. This is designed for squeezing +// declaration, specification, imports, and identifier lists. +func Squeeze[E ast.Node, S ~[]E](s S) S { + var zero E + count, dest := len(s), 0 + for src := 0; src < count; src++ { + if !reflect.DeepEqual(s[src], zero) { + // Swap the values, this will put the nil values to the end + // of the slice so that the tail isn't holding onto pointers. + s[dest], s[src] = s[src], s[dest] + dest++ + } + } + return s[:dest] +} + +type CallbackVisitor struct { + predicate func(node ast.Node) bool +} + +func NewCallbackVisitor(predicate func(node ast.Node) bool) *CallbackVisitor { + return &CallbackVisitor{predicate: predicate} +} + +func (v *CallbackVisitor) Visit(node ast.Node) ast.Visitor { + if v.predicate != nil && v.predicate(node) { + return v + } + v.predicate = nil + return nil +} diff --git a/compiler/astutil/astutil_test.go b/compiler/astutil/astutil_test.go index 4330fdb5a..c6ee71977 100644 --- a/compiler/astutil/astutil_test.go +++ b/compiler/astutil/astutil_test.go @@ -4,6 +4,7 @@ import ( "fmt" "go/ast" "go/token" + "strconv" "testing" "github.com/gopherjs/gopherjs/internal/srctesting" @@ -54,6 +55,47 @@ func TestImportsUnsafe(t *testing.T) { } } +func TestImportName(t *testing.T) { + tests := []struct { + desc string + src string + want string + }{ + { + desc: `named import`, + src: `import foo "some/other/bar"`, + want: `foo`, + }, { + desc: `unnamed import`, + src: `import "some/other/bar"`, + want: `bar`, + }, { + desc: `dot import`, + src: `import . "some/other/bar"`, + want: ``, + }, { + desc: `blank import`, + src: `import _ "some/other/bar"`, + want: ``, + }, + } + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + src := "package testpackage\n\n" + test.src + fset := token.NewFileSet() + file := srctesting.Parse(t, fset, src) + if len(file.Imports) != 1 { + t.Fatal(`expected one and only one import`) + } + importSpec := file.Imports[0] + got := ImportName(importSpec) + if got != test.want { + t.Fatalf(`ImportName() returned %q, want %q`, got, test.want) + } + }) + } +} + func TestFuncKey(t *testing.T) { tests := []struct { desc string @@ -101,54 +143,6 @@ func TestFuncKey(t *testing.T) { } } -func TestPruneOriginal(t *testing.T) { - tests := []struct { - desc string - src string - want bool - }{ - { - desc: "no comment", - src: `package testpackage; - func foo() {}`, - want: false, - }, { - desc: "regular godoc", - src: `package testpackage; - // foo does something - func foo() {}`, - want: false, - }, { - desc: "only directive", - src: `package testpackage; - //gopherjs:prune-original - func foo() {}`, - want: true, - }, { - desc: "directive with explanation", - src: `package testpackage; - //gopherjs:prune-original because reasons - func foo() {}`, - want: true, - }, { - desc: "directive in godoc", - src: `package testpackage; - // foo does something - //gopherjs:prune-original - func foo() {}`, - want: true, - }, - } - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - fdecl := srctesting.ParseFuncDecl(t, test.src) - if got := PruneOriginal(fdecl); got != test.want { - t.Errorf("PruneOriginal() returned %t, want %t", got, test.want) - } - }) - } -} - func TestHasDirectiveOnDecl(t *testing.T) { tests := []struct { desc string @@ -561,3 +555,61 @@ func TestEndsWithReturn(t *testing.T) { }) } } + +func TestSqueezeIdents(t *testing.T) { + tests := []struct { + desc string + count int + assign []int + }{ + { + desc: `no squeezing`, + count: 5, + assign: []int{0, 1, 2, 3, 4}, + }, { + desc: `missing front`, + count: 5, + assign: []int{3, 4}, + }, { + desc: `missing back`, + count: 5, + assign: []int{0, 1, 2}, + }, { + desc: `missing several`, + count: 10, + assign: []int{1, 2, 3, 6, 8}, + }, { + desc: `empty`, + count: 0, + assign: []int{}, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + input := make([]*ast.Ident, test.count) + for _, i := range test.assign { + input[i] = ast.NewIdent(strconv.Itoa(i)) + } + + result := Squeeze(input) + if len(result) != len(test.assign) { + t.Errorf("Squeeze() returned a slice %d long, want %d", len(result), len(test.assign)) + } + for i, id := range input { + if i < len(result) { + if id == nil { + t.Errorf(`Squeeze() returned a nil in result at %d`, i) + } else { + value, err := strconv.Atoi(id.Name) + if err != nil || value != test.assign[i] { + t.Errorf(`Squeeze() returned %s at %d instead of %d`, id.Name, i, test.assign[i]) + } + } + } else if id != nil { + t.Errorf(`Squeeze() didn't clear out tail of slice, want %d nil`, i) + } + } + }) + } +} From 84774ae19c6ae40efd143c915d4947a5166ca99a Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 19 Dec 2023 16:24:02 -0700 Subject: [PATCH 08/29] Changed to use Inspect --- compiler/astutil/astutil.go | 55 ++++++++++---------------------- compiler/astutil/astutil_test.go | 53 ++++++++++-------------------- 2 files changed, 32 insertions(+), 76 deletions(-) diff --git a/compiler/astutil/astutil.go b/compiler/astutil/astutil.go index c7481fd27..abe17d94a 100644 --- a/compiler/astutil/astutil.go +++ b/compiler/astutil/astutil.go @@ -97,41 +97,6 @@ func KeepOriginal(d *ast.FuncDecl) bool { return hasDirective(d, `keep-original`) } -// anyDocLine calls the given predicate on all associated documentation -// lines and line-comment lines from the given node. -// If the predicate returns true for any line then true is returned. -func anyDocLine(node any, predicate func(line string) bool) bool { - switch a := node.(type) { - case *ast.Comment: - return a != nil && predicate(a.Text) - case *ast.CommentGroup: - if a != nil { - for _, c := range a.List { - if anyDocLine(c, predicate) { - return true - } - } - } - return false - case *ast.Field: - return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) - case *ast.File: - return a != nil && anyDocLine(a.Doc, predicate) - case *ast.FuncDecl: - return a != nil && anyDocLine(a.Doc, predicate) - case *ast.GenDecl: - return a != nil && anyDocLine(a.Doc, predicate) - case *ast.ImportSpec: - return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) - case *ast.TypeSpec: - return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) - case *ast.ValueSpec: - return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) - default: - panic(fmt.Errorf(`unexpected node type to get doc from: %T`, node)) - } -} - // directiveMatcher is a regex which matches a GopherJS directive // and finds the directive action. var directiveMatcher = regexp.MustCompile(`^\/(?:\/|\*)gopherjs:([\w-]+)`) @@ -144,11 +109,23 @@ var directiveMatcher = regexp.MustCompile(`^\/(?:\/|\*)gopherjs:([\w-]+)`) // must be one or more letter, decimal, underscore, or hyphen. // // see https://pkg.go.dev/cmd/compile#hdr-Compiler_Directives -func hasDirective(node any, directiveAction string) bool { - return anyDocLine(node, func(line string) bool { - m := directiveMatcher.FindStringSubmatch(line) - return len(m) == 2 && m[1] == directiveAction +func hasDirective(node ast.Node, directiveAction string) bool { + foundDirective := false + ast.Inspect(node, func(n ast.Node) bool { + switch a := n.(type) { + case *ast.Comment: + m := directiveMatcher.FindStringSubmatch(a.Text) + if len(m) == 2 && m[1] == directiveAction { + foundDirective = true + } + return false + case *ast.CommentGroup: + return !foundDirective + default: + return n == node + } }) + return foundDirective } // FindLoopStmt tries to find the loop statement among the AST nodes in the diff --git a/compiler/astutil/astutil_test.go b/compiler/astutil/astutil_test.go index 61fc2b18c..80b689b5f 100644 --- a/compiler/astutil/astutil_test.go +++ b/compiler/astutil/astutil_test.go @@ -1,7 +1,6 @@ package astutil import ( - "fmt" "go/ast" "go/token" "testing" @@ -210,6 +209,16 @@ func TestHasDirectiveOnDecl(t *testing.T) { //gopherjs:do-stuff type Foo int`, want: true, + }, { + desc: `directive on specification, not on declaration`, + src: `package testpackage; + type ( + Foo int + + //gopherjs:do-stuff + Bar struct{} + )`, + want: false, }, { desc: `no directive on const declaration`, src: `package testpackage; @@ -268,6 +277,12 @@ func TestHasDirectiveOnSpec(t *testing.T) { src: `package testpackage; type Foo int`, want: false, + }, { + desc: `directive on declaration, not on specification`, + src: `package testpackage; + //gopherjs:do-stuff + type Foo int`, + want: false, }, { desc: `directive in doc on type specification`, src: `package testpackage; @@ -457,42 +472,6 @@ func TestHasDirectiveOnField(t *testing.T) { } } -func TestHasDirectiveBadCase(t *testing.T) { - tests := []struct { - desc string - node any - want string - }{ - { - desc: `untyped nil node`, - node: nil, - want: `unexpected node type to get doc from: `, - }, { - desc: `unexpected node type`, - node: &ast.ArrayType{}, - want: `unexpected node type to get doc from: *ast.ArrayType`, - }, { - desc: `nil expected node type`, - node: (*ast.FuncDecl)(nil), - want: ``, // no panic - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - const action = `do-stuff` - var got string - func() { - defer func() { got = fmt.Sprint(recover()) }() - hasDirective(test.node, action) - }() - if got != test.want { - t.Errorf(`hasDirective(%T, %q) returned %s, want %s`, test.node, action, got, test.want) - } - }) - } -} - func TestEndsWithReturn(t *testing.T) { tests := []struct { desc string From 7758c0d142d880a546e128b59f8e4b1bb79d5079 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 19 Dec 2023 16:46:10 -0700 Subject: [PATCH 09/29] Updated to use Inspect and srctesting --- build/build.go | 4 ++-- build/build_test.go | 22 +++++----------------- compiler/astutil/astutil.go | 16 ---------------- 3 files changed, 7 insertions(+), 35 deletions(-) diff --git a/build/build.go b/build/build.go index b32ed9f37..378fcc79f 100644 --- a/build/build.go +++ b/build/build.go @@ -406,14 +406,14 @@ func pruneImports(file *ast.File) { } // Remove "unused import" for any import which is used. - ast.Walk(astutil.NewCallbackVisitor(func(n ast.Node) bool { + ast.Inspect(file, func(n ast.Node) bool { if sel, ok := n.(*ast.SelectorExpr); ok { if id, ok := sel.X.(*ast.Ident); ok && id.Obj == nil { delete(unused, id.Name) } } return len(unused) > 0 - }), file) + }) if len(unused) == 0 { return } diff --git a/build/build_test.go b/build/build_test.go index 81a12b36d..8cb721554 100644 --- a/build/build_test.go +++ b/build/build_test.go @@ -1,10 +1,8 @@ package build import ( - "bytes" "fmt" gobuild "go/build" - "go/printer" "go/token" "strconv" "testing" @@ -390,16 +388,11 @@ func TestOverlayAugmentation(t *testing.T) { augmentOverlayFile(fileSrc, overrides) pruneImports(fileSrc) - buf := &bytes.Buffer{} - _ = printer.Fprint(buf, fsetSrc, fileSrc) - got := buf.String() + got := srctesting.Format(t, fsetSrc, fileSrc) fsetWant := token.NewFileSet() fileWant := srctesting.Parse(t, fsetWant, pkgName+test.want) - - buf.Reset() - _ = printer.Fprint(buf, fsetWant, fileWant) - want := buf.String() + want := srctesting.Format(t, fsetWant, fileWant) if got != want { t.Errorf("augmentOverlayFile and pruneImports got unexpected code:\n"+ @@ -536,7 +529,7 @@ func TestOriginalAugmentation(t *testing.T) { func NewFoo(bar int) *Foo { return &Foo{bar: bar} }`, // NewFoo is not removed automatically since - // only functions with Foo as a receiver is removed. + // only functions with Foo as a receiver are removed. want: `func NewFoo(bar int) *Foo { return &Foo{bar: bar} }`, }, { desc: `remove generics`, @@ -591,16 +584,11 @@ func TestOriginalAugmentation(t *testing.T) { augmentOriginalFile(fileSrc, test.info) pruneImports(fileSrc) - buf := &bytes.Buffer{} - _ = printer.Fprint(buf, fsetSrc, fileSrc) - got := buf.String() + got := srctesting.Format(t, fsetSrc, fileSrc) fsetWant := token.NewFileSet() fileWant := srctesting.Parse(t, fsetWant, pkgName+test.want) - - buf.Reset() - _ = printer.Fprint(buf, fsetWant, fileWant) - want := buf.String() + want := srctesting.Format(t, fsetWant, fileWant) if got != want { t.Errorf("augmentOriginalImports, augmentOriginalFile, and pruneImports got unexpected code:\n"+ diff --git a/compiler/astutil/astutil.go b/compiler/astutil/astutil.go index 18d693f0a..1f7196766 100644 --- a/compiler/astutil/astutil.go +++ b/compiler/astutil/astutil.go @@ -251,19 +251,3 @@ func Squeeze[E ast.Node, S ~[]E](s S) S { } return s[:dest] } - -type CallbackVisitor struct { - predicate func(node ast.Node) bool -} - -func NewCallbackVisitor(predicate func(node ast.Node) bool) *CallbackVisitor { - return &CallbackVisitor{predicate: predicate} -} - -func (v *CallbackVisitor) Visit(node ast.Node) ast.Visitor { - if v.predicate != nil && v.predicate(node) { - return v - } - v.predicate = nil - return nil -} From e03bfea71a099297df61b2371ccf0b5b5779313e Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 16 Jan 2024 14:18:42 -0700 Subject: [PATCH 10/29] Update compiler for go1.19 --- compiler/analysis/info.go | 5 ++++- compiler/expressions.go | 11 +++++++---- compiler/package.go | 30 +++++++++++++++++------------- compiler/statements.go | 8 ++++---- compiler/utils.go | 14 +++++++------- 5 files changed, 39 insertions(+), 29 deletions(-) diff --git a/compiler/analysis/info.go b/compiler/analysis/info.go index c984b726f..304c8808a 100644 --- a/compiler/analysis/info.go +++ b/compiler/analysis/info.go @@ -93,7 +93,10 @@ func (info *Info) newFuncInfo(n ast.Node) *FuncInfo { } func (info *Info) IsBlocking(fun *types.Func) bool { - return len(info.FuncDeclInfos[fun].Blocking) > 0 + if funInfo := info.FuncDeclInfos[fun]; funInfo != nil { + return len(funInfo.Blocking) > 0 + } + panic(fmt.Errorf(`info did not have function declaration for %s`, fun.FullName())) } func AnalyzePkg(files []*ast.File, fileSet *token.FileSet, typesInfo *types.Info, typesPkg *types.Package, isBlocking func(*types.Func) bool) *Info { diff --git a/compiler/expressions.go b/compiler/expressions.go index db08cc31f..21971ab5f 100644 --- a/compiler/expressions.go +++ b/compiler/expressions.go @@ -267,7 +267,7 @@ func (fc *funcContext) translateExpr(expr ast.Expr) *expression { case token.ARROW: call := &ast.CallExpr{ - Fun: fc.newIdent("$recv", types.NewSignature(nil, types.NewTuple(types.NewVar(0, nil, "", t)), types.NewTuple(types.NewVar(0, nil, "", exprType), types.NewVar(0, nil, "", types.Typ[types.Bool])), false)), + Fun: fc.newIdent("$recv", types.NewSignatureType(nil, nil, nil, types.NewTuple(types.NewVar(0, nil, "", t)), types.NewTuple(types.NewVar(0, nil, "", exprType), types.NewVar(0, nil, "", types.Typ[types.Bool])), false)), Args: []ast.Expr{e.X}, } fc.Blocking[call] = true @@ -520,8 +520,11 @@ func (fc *funcContext) translateExpr(expr ast.Expr) *expression { ) case *types.Basic: return fc.formatExpr("%e.charCodeAt(%f)", e.X, e.Index) + case *types.Signature: + err := bailout(fmt.Errorf(`unsupported type parameters used at %s`, fc.pkgCtx.fileSet.Position(e.Pos()))) + panic(err) default: - panic(fmt.Sprintf("Unhandled IndexExpr: %T\n", t)) + panic(fmt.Errorf(`unhandled IndexExpr: %T`, t)) } case *ast.SliceExpr: @@ -703,7 +706,7 @@ func (fc *funcContext) translateExpr(expr ast.Expr) *expression { case "Float": return fc.internalize(recv, types.Typ[types.Float64]) case "Interface": - return fc.internalize(recv, types.NewInterface(nil, nil)) + return fc.internalize(recv, types.NewInterfaceType(nil, nil)) case "Unsafe": return recv default: @@ -998,7 +1001,7 @@ func (fc *funcContext) translateBuiltin(name string, sig *types.Signature, args panic(fmt.Sprintf("Unhandled cap type: %T\n", argType)) } case "panic": - return fc.formatExpr("$panic(%s)", fc.translateImplicitConversion(args[0], types.NewInterface(nil, nil))) + return fc.formatExpr("$panic(%s)", fc.translateImplicitConversion(args[0], types.NewInterfaceType(nil, nil))) case "append": if ellipsis || len(args) == 1 { argStr := fc.translateArgs(sig, args, ellipsis) diff --git a/compiler/package.go b/compiler/package.go index 93c22f1c5..ad918ba3e 100644 --- a/compiler/package.go +++ b/compiler/package.go @@ -131,6 +131,7 @@ func Compile(importPath string, files []*ast.File, fileSet *token.FileSet, impor } if fe, ok := bailingOut(e); ok { // Orderly bailout, return whatever clues we already have. + fmt.Fprintf(fe, `building package %q`, importPath) err = fe return } @@ -269,7 +270,7 @@ func Compile(importPath string, files []*ast.File, fileSet *token.FileSet, impor } sort.Strings(importedPaths) for _, impPath := range importedPaths { - id := funcCtx.newIdent(fmt.Sprintf(`%s.$init`, funcCtx.pkgCtx.pkgVars[impPath]), types.NewSignature(nil, nil, nil, false)) + id := funcCtx.newIdent(fmt.Sprintf(`%s.$init`, funcCtx.pkgCtx.pkgVars[impPath]), types.NewSignatureType(nil, nil, nil, nil, nil, false)) call := &ast.CallExpr{Fun: id} funcCtx.Blocking[call] = true funcCtx.Flattened[call] = true @@ -287,13 +288,6 @@ func Compile(importPath string, files []*ast.File, fileSet *token.FileSet, impor switch d := decl.(type) { case *ast.FuncDecl: sig := funcCtx.pkgCtx.Defs[d.Name].(*types.Func).Type().(*types.Signature) - var recvType types.Type - if sig.Recv() != nil { - recvType = sig.Recv().Type() - if ptr, isPtr := recvType.(*types.Pointer); isPtr { - recvType = ptr.Elem() - } - } if sig.Recv() == nil { funcCtx.objectName(funcCtx.pkgCtx.Defs[d.Name].(*types.Func)) // register toplevel name } @@ -421,7 +415,7 @@ func Compile(importPath string, files []*ast.File, fileSet *token.FileSet, impor d.DceObjectFilter = "" case "init": d.InitCode = funcCtx.CatchOutput(1, func() { - id := funcCtx.newIdent("", types.NewSignature(nil, nil, nil, false)) + id := funcCtx.newIdent("", types.NewSignatureType(nil, nil, nil, nil, nil, false)) funcCtx.pkgCtx.Uses[id] = o call := &ast.CallExpr{Fun: id} if len(funcCtx.pkgCtx.FuncDeclInfos[o].Blocking) != 0 { @@ -438,7 +432,14 @@ func Compile(importPath string, files []*ast.File, fileSet *token.FileSet, impor if isPointer { namedRecvType = ptr.Elem().(*types.Named) } - d.NamedRecvType = funcCtx.objectName(namedRecvType.Obj()) + if namedRecvType.TypeParams() != nil { + return nil, scanner.Error{ + Pos: fileSet.Position(o.Pos()), + Msg: fmt.Sprintf("type %s: type parameters are not supported by GopherJS: https://github.com/gopherjs/gopherjs/issues/1013", o.FullName()), + } + } + name := funcCtx.objectName(namedRecvType.Obj()) + d.NamedRecvType = name d.DceObjectFilter = namedRecvType.Obj().Name() if !fun.Name.IsExported() { d.DceMethodFilter = o.Name() + "~" @@ -454,7 +455,7 @@ func Compile(importPath string, files []*ast.File, fileSet *token.FileSet, impor if mainFunc == nil { return nil, fmt.Errorf("missing main function") } - id := funcCtx.newIdent("", types.NewSignature(nil, nil, nil, false)) + id := funcCtx.newIdent("", types.NewSignatureType(nil, nil, nil, nil, nil, false)) funcCtx.pkgCtx.Uses[id] = mainFunc call := &ast.CallExpr{Fun: id} ifStmt := &ast.IfStmt{ @@ -636,9 +637,9 @@ func (fc *funcContext) initArgs(ty types.Type) string { case *types.Map: return fmt.Sprintf("%s, %s", fc.typeName(t.Key()), fc.typeName(t.Elem())) case *types.Pointer: - return fmt.Sprintf("%s", fc.typeName(t.Elem())) + return fc.typeName(t.Elem()) case *types.Slice: - return fmt.Sprintf("%s", fc.typeName(t.Elem())) + return fc.typeName(t.Elem()) case *types.Signature: params := make([]string, t.Params().Len()) for i := range params { @@ -660,6 +661,9 @@ func (fc *funcContext) initArgs(ty types.Type) string { fields[i] = fmt.Sprintf(`{prop: "%s", name: %s, embedded: %t, exported: %t, typ: %s, tag: %s}`, fieldName(t, i), encodeString(field.Name()), field.Anonymous(), field.Exported(), fc.typeName(field.Type()), encodeString(t.Tag(i))) } return fmt.Sprintf(`"%s", [%s]`, pkgPath, strings.Join(fields, ", ")) + case *types.TypeParam: + err := bailout(fmt.Errorf(`%v has unexpected generic type parameter %T`, ty, ty)) + panic(err) default: err := bailout(fmt.Errorf("%v has unexpected type %T", ty, ty)) panic(err) diff --git a/compiler/statements.go b/compiler/statements.go index f8d791948..8518f9b71 100644 --- a/compiler/statements.go +++ b/compiler/statements.go @@ -32,7 +32,7 @@ func (fc *funcContext) translateStmt(stmt ast.Stmt, label *types.Label) { panic(err) // Continue orderly bailout. } - // Oh noes, we've tried to compile something so bad that compiler paniced + // Oh noes, we've tried to compile something so bad that compiler panicked // and ran away. Let's gather some debugging clues. bail := bailout(err) pos := stmt.Pos() @@ -471,7 +471,7 @@ func (fc *funcContext) translateStmt(stmt ast.Stmt, label *types.Label) { case *ast.SendStmt: chanType := fc.pkgCtx.TypeOf(s.Chan).Underlying().(*types.Chan) call := &ast.CallExpr{ - Fun: fc.newIdent("$send", types.NewSignature(nil, types.NewTuple(types.NewVar(0, nil, "", chanType), types.NewVar(0, nil, "", chanType.Elem())), nil, false)), + Fun: fc.newIdent("$send", types.NewSignatureType(nil, nil, nil, types.NewTuple(types.NewVar(0, nil, "", chanType), types.NewVar(0, nil, "", chanType.Elem())), nil, false)), Args: []ast.Expr{s.Chan, fc.newIdent(fc.translateImplicitConversionWithCloning(s.Value, chanType.Elem()).String(), chanType.Elem())}, } fc.Blocking[call] = true @@ -522,8 +522,8 @@ func (fc *funcContext) translateStmt(stmt ast.Stmt, label *types.Label) { } selectCall := fc.setType(&ast.CallExpr{ - Fun: fc.newIdent("$select", types.NewSignature(nil, types.NewTuple(types.NewVar(0, nil, "", types.NewInterface(nil, nil))), types.NewTuple(types.NewVar(0, nil, "", types.Typ[types.Int])), false)), - Args: []ast.Expr{fc.newIdent(fmt.Sprintf("[%s]", strings.Join(channels, ", ")), types.NewInterface(nil, nil))}, + Fun: fc.newIdent("$select", types.NewSignatureType(nil, nil, nil, types.NewTuple(types.NewVar(0, nil, "", types.NewInterfaceType(nil, nil))), types.NewTuple(types.NewVar(0, nil, "", types.Typ[types.Int])), false)), + Args: []ast.Expr{fc.newIdent(fmt.Sprintf("[%s]", strings.Join(channels, ", ")), types.NewInterfaceType(nil, nil))}, }, types.Typ[types.Int]) if !hasDefault { fc.Blocking[selectCall] = true diff --git a/compiler/utils.go b/compiler/utils.go index 8b18c29c1..058437f67 100644 --- a/compiler/utils.go +++ b/compiler/utils.go @@ -518,13 +518,13 @@ func isBlank(expr ast.Expr) bool { // // For example, consider a Go type: // -// type SecretInt int -// func (_ SecretInt) String() string { return "" } +// type SecretInt int +// func (_ SecretInt) String() string { return "" } // -// func main() { -// var i SecretInt = 1 -// println(i.String()) -// } +// func main() { +// var i SecretInt = 1 +// println(i.String()) +// } // // For this example the compiler will generate code similar to the snippet below: // @@ -765,7 +765,7 @@ func (st signatureTypes) Param(i int, ellipsis bool) types.Type { } if !st.Sig.Variadic() { // This should never happen if the code was type-checked successfully. - panic(fmt.Errorf("Tried to access parameter %d of a non-variadic signature %s", i, st.Sig)) + panic(fmt.Errorf("tried to access parameter %d of a non-variadic signature %s", i, st.Sig)) } if ellipsis { return st.VariadicType() From 5f9eeb8ca88cd5a1215840df285cc3fc77825c3b Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 16 Jan 2024 13:40:39 -0700 Subject: [PATCH 11/29] Fix Build Issue --- build/build.go | 82 ++++++++++++++++++++++++--- build/build_test.go | 109 +++++++++++++++++++++++++++++++++++- compiler/astutil/astutil.go | 32 +++++++++++ 3 files changed, 212 insertions(+), 11 deletions(-) diff --git a/build/build.go b/build/build.go index 378fcc79f..30e2b15aa 100644 --- a/build/build.go +++ b/build/build.go @@ -130,6 +130,11 @@ type overrideInfo struct { // If the method is defined in the overlays and therefore has its // own overrides, this will be ignored. purgeMethods bool + + // overrideSignature is the function definition given in the overlays + // that should be used to replace the signature in the originals. + // Only receivers, type parameters, parameters, and results will be used. + overrideSignature *ast.FuncDecl } // parseAndAugment parses and returns all .go files of given pkg. @@ -270,9 +275,14 @@ func augmentOverlayFile(file *ast.File, overrides map[string]overrideInfo) { switch d := decl.(type) { case *ast.FuncDecl: k := astutil.FuncKey(d) - overrides[k] = overrideInfo{ + oi := overrideInfo{ keepOriginal: astutil.KeepOriginal(d), } + if astutil.OverrideSignature(d) { + oi.overrideSignature = d + purgeDecl = true + } + overrides[k] = oi case *ast.GenDecl: for j, spec := range d.Specs { purgeSpec := purgeDecl || astutil.Purge(spec) @@ -323,11 +333,21 @@ func augmentOriginalFile(file *ast.File, overrides map[string]overrideInfo) { switch d := decl.(type) { case *ast.FuncDecl: if info, ok := overrides[astutil.FuncKey(d)]; ok { + removeFunc := true if info.keepOriginal { // Allow overridden function calls // The standard library implementation of foo() becomes _gopherjs_original_foo() d.Name.Name = "_gopherjs_original_" + d.Name.Name - } else { + removeFunc = false + } + if overSig := info.overrideSignature; overSig != nil { + d.Recv = overSig.Recv + d.Type.TypeParams = overSig.Type.TypeParams + d.Type.Params = overSig.Type.Params + d.Type.Results = overSig.Type.Results + removeFunc = false + } + if removeFunc { file.Decls[i] = nil } } else if recvKey := astutil.FuncReceiverKey(d); len(recvKey) > 0 { @@ -391,13 +411,32 @@ func augmentOriginalFile(file *ast.File, overrides map[string]overrideInfo) { finalizeRemovals(file) } +// isOnlyImports determines if this file is empty except for imports. +func isOnlyImports(file *ast.File) bool { + for _, decl := range file.Decls { + if gen, ok := decl.(*ast.GenDecl); !ok || gen.Tok != token.IMPORT { + return false + } + } + return true +} + // pruneImports will remove any unused imports from the file. // -// This will not remove any dot (`.`) or blank (`_`) imports. +// This will not remove any dot (`.`) or blank (`_`) imports, unless +// there are no declarations or directives meaning that all the imports +// should be cleared. // If the removal of code causes an import to be removed, the init's from that // import may not be run anymore. If we still need to run an init for an import // which is no longer used, add it to the overlay as a blank (`_`) import. func pruneImports(file *ast.File) { + if isOnlyImports(file) && !astutil.HasDirectivePrefix(file, `//go:linkname `) { + // The file is empty, remove all imports including any `.` or `_` imports. + file.Imports = nil + file.Decls = nil + return + } + unused := make(map[string]int, len(file.Imports)) for i, in := range file.Imports { if name := astutil.ImportName(in); len(name) > 0 { @@ -405,7 +444,7 @@ func pruneImports(file *ast.File) { } } - // Remove "unused import" for any import which is used. + // Remove "unused imports" for any import which is used. ast.Inspect(file, func(n ast.Node) bool { if sel, ok := n.(*ast.SelectorExpr); ok { if id, ok := sel.X.(*ast.Ident); ok && id.Obj == nil { @@ -418,6 +457,24 @@ func pruneImports(file *ast.File) { return } + // Remove "unused imports" for any import used for a directive. + directiveImports := map[string]string{ + `unsafe`: `//go:linkname `, + `embed`: `//go:embed `, + } + for name, index := range unused { + in := file.Imports[index] + path, _ := strconv.Unquote(in.Path.Value) + directivePrefix, hasPath := directiveImports[path] + if hasPath && astutil.HasDirectivePrefix(file, directivePrefix) { + delete(unused, name) + if len(unused) == 0 { + return + } + break + } + } + // Remove all unused import specifications isUnusedSpec := map[*ast.ImportSpec]bool{} for _, index := range unused { @@ -442,9 +499,8 @@ func pruneImports(file *ast.File) { } // finalizeRemovals fully removes any declaration, specification, imports -// that have been set to nil. This will also remove the file's top-level -// comment group to remove any unassociated comments, including the comments -// from removed code. +// that have been set to nil. This will also remove any unassociated comment +// groups, including the comments from removed code. func finalizeRemovals(file *ast.File) { fileChanged := false for i, decl := range file.Decls { @@ -487,8 +543,18 @@ func finalizeRemovals(file *ast.File) { if fileChanged { file.Decls = astutil.Squeeze(file.Decls) } + file.Imports = astutil.Squeeze(file.Imports) - file.Comments = nil + + file.Comments = nil // clear this first so ast.Inspect doesn't walk it. + remComments := []*ast.CommentGroup{} + ast.Inspect(file, func(n ast.Node) bool { + if cg, ok := n.(*ast.CommentGroup); ok { + remComments = append(remComments, cg) + } + return true + }) + file.Comments = remComments } // Options controls build process behavior. diff --git a/build/build_test.go b/build/build_test.go index 8cb721554..f54281cb8 100644 --- a/build/build_test.go +++ b/build/build_test.go @@ -371,6 +371,36 @@ func TestOverlayAugmentation(t *testing.T) { `Sort`: {}, `Equal`: {}, }, + }, { + desc: `remove unsafe and embed if not needed`, + src: `import "unsafe" + import "embed" + + //gopherjs:purge + var eFile embed.FS + + //gopherjs:purge + func SwapPointer(addr *unsafe.Pointer, new unsafe.Pointer) (old unsafe.Pointer)`, + want: ``, + expInfo: map[string]overrideInfo{ + `SwapPointer`: {}, + `eFile`: {}, + }, + }, { + desc: `keep unsafe and embed for directives`, + src: `import "unsafe" + import "embed" + + //go:embed hello.txt + var eFile embed.FS + + //go:linkname runtimeNano runtime.nanotime + func runtimeNano() int64`, + noCodeChange: true, + expInfo: map[string]overrideInfo{ + `eFile`: {}, + `runtimeNano`: {}, + }, }, } @@ -539,6 +569,9 @@ func TestOriginalAugmentation(t *testing.T) { `Equal`: {}, }, src: `import "cmp" + + // keeps the isOnlyImports from skipping what is being tested. + func foo() {} type Pointer[T any] struct {} @@ -546,7 +579,8 @@ func TestOriginalAugmentation(t *testing.T) { // overlay had stub "func Equal() {}" func Equal[S ~[]E, E any](s1, s2 S) bool {}`, - want: ``, + want: `// keeps the isOnlyImports from skipping what is being tested. + func foo() {}`, }, { desc: `purge generics`, info: map[string]overrideInfo{ @@ -556,6 +590,9 @@ func TestOriginalAugmentation(t *testing.T) { }, src: `import "cmp" + // keeps the isOnlyImports from skipping what is being tested. + func foo() {} + type Pointer[T any] struct {} func (x *Pointer[T]) Load() *T {} func (x *Pointer[T]) Store(val *T) {} @@ -564,12 +601,78 @@ func TestOriginalAugmentation(t *testing.T) { // overlay had stub "func Equal() {}" func Equal[S ~[]E, E any](s1, s2 S) bool {}`, - want: ``, + want: `// keeps the isOnlyImports from skipping what is being tested. + func foo() {}`, }, { desc: `prune an unused import`, info: map[string]overrideInfo{}, - src: `import foo "some/other/bar"`, + src: `import foo "some/other/bar" + + // keeps the isOnlyImports from skipping what is being tested. + func foo() {}`, + want: `// keeps the isOnlyImports from skipping what is being tested. + func foo() {}`, + }, { + desc: `override signature of function`, + info: map[string]overrideInfo{ + `Foo`: { + overrideSignature: srctesting.ParseFuncDecl(t, + `package whatever + func Foo(a, b any) (any, bool) {}`), + }, + }, + src: `func Foo[T comparable](a, b T) (T, bool) { + if a == b { + return a, true + } + return b, false + }`, + want: `func Foo(a, b any) (any, bool) { + if a == b { + return a, true + } + return b, false + }`, + }, { + desc: `override signature of method`, + info: map[string]overrideInfo{ + `Foo.Bar`: { + overrideSignature: srctesting.ParseFuncDecl(t, + `package whatever + func (r *Foo) Bar(a, b any) (any, bool) {}`), + }, + }, + src: `func (r *Foo[T]) Bar(a, b T) (T, bool) { + if r.isSame(a, b) { + return a, true + } + return b, false + }`, + want: `func (r *Foo) Bar(a, b any) (any, bool) { + if r.isSame(a, b) { + return a, true + } + return b, false + }`, + }, { + desc: `empty file removes all imports`, + info: map[string]overrideInfo{ + `foo`: {}, + }, + src: `import . "math/rand" + func foo() int { + return Int() + }`, want: ``, + }, { + desc: `empty file with directive`, + info: map[string]overrideInfo{ + `foo`: {}, + }, + src: `//go:linkname foo bar + import _ "unsafe"`, + want: `//go:linkname foo bar + import _ "unsafe"`, }, } diff --git a/compiler/astutil/astutil.go b/compiler/astutil/astutil.go index 1f7196766..5cfe2dbd3 100644 --- a/compiler/astutil/astutil.go +++ b/compiler/astutil/astutil.go @@ -9,6 +9,7 @@ import ( "reflect" "regexp" "strconv" + "strings" ) func RemoveParens(e ast.Expr) ast.Expr { @@ -148,6 +149,24 @@ func Purge(d ast.Node) bool { return hasDirective(d, `purge`) } +// OverrideSignature returns true if gopherjs:override-signature directive is +// present on a function. +// +// `//gopherjs:override-signature` is a GopherJS-specific directive, which can +// be applied in native overlays and will instruct the augmentation logic to +// replace the original function signature which has the same FuncKey with the +// signature defined in the native overlays. +// This directive can be used to remove generics from a function signature or +// to replace a receiver of a function with another one. The given native +// overlay function will be removed, so no method body is needed in the overlay. +// +// The new signature may not contain types which require a new import since +// the imports will not be automatically added when needed, only removed. +// Use a type alias in the overlay to deal manage imports. +func OverrideSignature(d *ast.FuncDecl) bool { + return hasDirective(d, `override-signature`) +} + // directiveMatcher is a regex which matches a GopherJS directive // and finds the directive action. var directiveMatcher = regexp.MustCompile(`^\/(?:\/|\*)gopherjs:([\w-]+)`) @@ -179,6 +198,19 @@ func hasDirective(node ast.Node, directiveAction string) bool { return foundDirective } +// HasDirectivePrefix determines if any line in the given file +// has the given directive prefix in it. +func HasDirectivePrefix(file *ast.File, prefix string) bool { + for _, cg := range file.Comments { + for _, c := range cg.List { + if strings.HasPrefix(c.Text, prefix) { + return true + } + } + } + return false +} + // FindLoopStmt tries to find the loop statement among the AST nodes in the // |stack| that corresponds to the break/continue statement represented by // branch. From a83c5fa72a01c93c19077fedec6fb5c2fa965915 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 16 Jan 2024 15:06:25 -0700 Subject: [PATCH 12/29] Updating natives for go1.19 --- .../natives/src/crypto/elliptic/nistec.go | 81 ++++++++ .../src/crypto/internal/boring/bbig/big.go | 42 ++++ .../src/crypto/internal/nistec/nistec_test.go | 88 +++++++++ .../src/crypto/internal/nistec/wrapper.go | 185 ++++++++++++++++++ compiler/natives/src/go/token/position.go | 26 +++ compiler/natives/src/hash/maphash/maphash.go | 2 +- compiler/natives/src/runtime/runtime.go | 3 +- compiler/natives/src/sync/atomic/atomic.go | 3 + .../natives/src/sync/atomic/atomic_test.go | 53 ++++- compiler/natives/src/sync/sync.go | 13 +- compiler/natives/src/testing/helper_test.go | 4 + .../natives/src/testing/helperfuncs_test.go | 13 ++ 12 files changed, 504 insertions(+), 9 deletions(-) create mode 100644 compiler/natives/src/crypto/elliptic/nistec.go create mode 100644 compiler/natives/src/crypto/internal/boring/bbig/big.go create mode 100644 compiler/natives/src/crypto/internal/nistec/nistec_test.go create mode 100644 compiler/natives/src/crypto/internal/nistec/wrapper.go create mode 100644 compiler/natives/src/go/token/position.go create mode 100644 compiler/natives/src/testing/helperfuncs_test.go diff --git a/compiler/natives/src/crypto/elliptic/nistec.go b/compiler/natives/src/crypto/elliptic/nistec.go new file mode 100644 index 000000000..326c602d5 --- /dev/null +++ b/compiler/natives/src/crypto/elliptic/nistec.go @@ -0,0 +1,81 @@ +//go:build js +// +build js + +package elliptic + +import ( + "crypto/internal/nistec" + "math/big" +) + +// nistPoint uses generics so must be removed for generic-less GopherJS. +// All the following code changes in this file are to make p224, p256, +// p521, and p384 still function correctly without this generic struct. +// +//gopherjs:purge for go1.19 without generics +type nistPoint[T any] interface{} + +// nistCurve replaces the generics with a version using the wrappedPoint +// interface, then update all the method signatures to also use wrappedPoint. +type nistCurve struct { + newPoint func() nistec.WrappedPoint + params *CurveParams +} + +//gopherjs:override-signature +func (curve *nistCurve) Params() *CurveParams + +//gopherjs:override-signature +func (curve *nistCurve) IsOnCurve(x, y *big.Int) bool + +//gopherjs:override-signature +func (curve *nistCurve) pointFromAffine(x, y *big.Int) (p nistec.WrappedPoint, err error) + +//gopherjs:override-signature +func (curve *nistCurve) pointToAffine(p nistec.WrappedPoint) (x, y *big.Int) + +//gopherjs:override-signature +func (curve *nistCurve) Add(x1, y1, x2, y2 *big.Int) (*big.Int, *big.Int) + +//gopherjs:override-signature +func (curve *nistCurve) Double(x1, y1 *big.Int) (*big.Int, *big.Int) + +//gopherjs:override-signature +func (curve *nistCurve) normalizeScalar(scalar []byte) []byte + +//gopherjs:override-signature +func (curve *nistCurve) ScalarMult(Bx, By *big.Int, scalar []byte) (*big.Int, *big.Int) + +//gopherjs:override-signature +func (curve *nistCurve) ScalarBaseMult(scalar []byte) (*big.Int, *big.Int) + +//gopherjs:override-signature +func (curve *nistCurve) CombinedMult(Px, Py *big.Int, s1, s2 []byte) (x, y *big.Int) + +//gopherjs:override-signature +func (curve *nistCurve) Unmarshal(data []byte) (x, y *big.Int) + +//gopherjs:override-signature +func (curve *nistCurve) UnmarshalCompressed(data []byte) (x, y *big.Int) + +var p224 = &nistCurve{ + newPoint: nistec.NewP224WrappedPoint, +} + +type p256Curve struct { + nistCurve +} + +var p256 = &p256Curve{ + nistCurve: nistCurve{ + newPoint: nistec.NewP256WrappedPoint, + }, +} + +var p521 = &nistCurve{ + newPoint: nistec.NewP521WrappedPoint, +} + +var p384 = &nistCurve{ + newPoint: nistec.NewP384WrappedPoint, +} diff --git a/compiler/natives/src/crypto/internal/boring/bbig/big.go b/compiler/natives/src/crypto/internal/boring/bbig/big.go new file mode 100644 index 000000000..30ffe1fcd --- /dev/null +++ b/compiler/natives/src/crypto/internal/boring/bbig/big.go @@ -0,0 +1,42 @@ +//go:build js +// +build js + +package bbig + +import ( + "crypto/internal/boring" + "math/big" +) + +func Enc(b *big.Int) boring.BigInt { + if b == nil { + return nil + } + x := b.Bits() + if len(x) == 0 { + return boring.BigInt{} + } + // Replacing original which uses unsafe: + // return unsafe.Slice((*uint)(&x[0]), len(x)) + b2 := make(boring.BigInt, len(x)) + for i, w := range x { + b2[i] = uint(w) + } + return b2 +} + +func Dec(b boring.BigInt) *big.Int { + if b == nil { + return nil + } + if len(b) == 0 { + return new(big.Int) + } + // Replacing original which uses unsafe: + //x := unsafe.Slice((*big.Word)(&b[0]), len(b)) + x := make([]big.Word, len(b)) + for i, w := range b { + x[i] = big.Word(w) + } + return new(big.Int).SetBits(x) +} diff --git a/compiler/natives/src/crypto/internal/nistec/nistec_test.go b/compiler/natives/src/crypto/internal/nistec/nistec_test.go new file mode 100644 index 000000000..f89aaeae0 --- /dev/null +++ b/compiler/natives/src/crypto/internal/nistec/nistec_test.go @@ -0,0 +1,88 @@ +//go:build js +// +build js + +package nistec_test + +import ( + "crypto/elliptic" + "testing" +) + +func TestAllocations(t *testing.T) { + t.Skip("testing.AllocsPerRun not supported in GopherJS") +} + +//gopherjs:purge +type nistPoint[T any] interface{} + +func TestEquivalents(t *testing.T) { + t.Run("P224", func(t *testing.T) { + testEquivalents(t, nistec.NewP224WrappedPoint, nistec.NewP224WrappedGenerator, elliptic.P224()) + }) + t.Run("P256", func(t *testing.T) { + testEquivalents(t, nistec.NewP256WrappedPoint, nistec.NewP256WrappedGenerator, elliptic.P256()) + }) + t.Run("P384", func(t *testing.T) { + testEquivalents(t, nistec.NewP384WrappedPoint, nistec.NewP384WrappedGenerator, elliptic.P384()) + }) + t.Run("P521", func(t *testing.T) { + testEquivalents(t, nistec.NewP521WrappedPoint, nistec.NewP521WrappedGenerator, elliptic.P521()) + }) +} + +//gopherjs:override-signature +func testEquivalents(t *testing.T, newPoint, newGenerator func() WrappedPoint, c elliptic.Curve) {} + +func TestScalarMult(t *testing.T) { + t.Run("P224", func(t *testing.T) { + testScalarMult(t, nistec.NewP224WrappedPoint, nistec.NewP224WrappedGenerator, elliptic.P224()) + }) + t.Run("P256", func(t *testing.T) { + testScalarMult(t, nistec.NewP256WrappedPoint, nistec.NewP256WrappedGenerator, elliptic.P256()) + }) + t.Run("P384", func(t *testing.T) { + testScalarMult(t, nistec.NewP384WrappedPoint, nistec.NewP384WrappedGenerator, elliptic.P384()) + }) + t.Run("P521", func(t *testing.T) { + testScalarMult(t, nistec.NewP521WrappedPoint, nistec.NewP521WrappedGenerator, elliptic.P521()) + }) +} + +//gopherjs:override-signature +func testScalarMult(t *testing.T, newPoint, newGenerator func() WrappedPoint, c elliptic.Curve) + +func BenchmarkScalarMult(b *testing.B) { + b.Run("P224", func(b *testing.B) { + benchmarkScalarMult(b, nistec.NewP224WrappedGenerator(), 28) + }) + b.Run("P256", func(b *testing.B) { + benchmarkScalarMult(b, nistec.NewP256GWrappedenerator(), 32) + }) + b.Run("P384", func(b *testing.B) { + benchmarkScalarMult(b, nistec.NewP384WrappedGenerator(), 48) + }) + b.Run("P521", func(b *testing.B) { + benchmarkScalarMult(b, nistec.NewP521WrappedGenerator(), 66) + }) +} + +//gopherjs:override-signature +func benchmarkScalarMult(b *testing.B, p WrappedPoint, scalarSize int) + +func BenchmarkScalarBaseMult(b *testing.B) { + b.Run("P224", func(b *testing.B) { + benchmarkScalarBaseMult(b, nistec.NewP22Wrapped4Generator(), 28) + }) + b.Run("P256", func(b *testing.B) { + benchmarkScalarBaseMult(b, nistec.NewP256WrappedGenerator(), 32) + }) + b.Run("P384", func(b *testing.B) { + benchmarkScalarBaseMult(b, nistec.NewP384WrappedGenerator(), 48) + }) + b.Run("P521", func(b *testing.B) { + benchmarkScalarBaseMult(b, nistec.NewP521GWrappedenerator(), 66) + }) +} + +//gopherjs:override-signature +func benchmarkScalarBaseMult(b *testing.B, p WrappedPoint, scalarSize int) diff --git a/compiler/natives/src/crypto/internal/nistec/wrapper.go b/compiler/natives/src/crypto/internal/nistec/wrapper.go new file mode 100644 index 000000000..0d6706b52 --- /dev/null +++ b/compiler/natives/src/crypto/internal/nistec/wrapper.go @@ -0,0 +1,185 @@ +//go:build js +// +build js + +package nistec + +type WrappedPoint interface { + Bytes() []byte + SetBytes(b []byte) (WrappedPoint, error) + Add(w1, w2 WrappedPoint) WrappedPoint + Double(w1 WrappedPoint) WrappedPoint + ScalarMult(w1 WrappedPoint, scalar []byte) (WrappedPoint, error) + ScalarBaseMult(scalar []byte) (WrappedPoint, error) +} + +type p224Wrapper struct { + point *P224Point +} + +func wrapP224(point *P224Point) WrappedPoint { + return p224Wrapper{point: point} +} + +func NewP224WrappedPoint() WrappedPoint { + return wrapP224(NewP224Point()) +} + +func NewP224WrappedGenerator() WrappedPoint { + return wrapP224(NewP224Generator()) +} + +func (w p224Wrapper) Bytes() []byte { + return w.point.Bytes() +} + +func (w p224Wrapper) SetBytes(b []byte) (WrappedPoint, error) { + p, err := w.point.SetBytes(b) + return wrapP224(p), err +} + +func (w p224Wrapper) Add(w1, w2 WrappedPoint) WrappedPoint { + return wrapP224(w.point.Add(w1.(p224Wrapper).point, w2.(p224Wrapper).point)) +} + +func (w p224Wrapper) Double(w1 WrappedPoint) WrappedPoint { + return wrapP224(w.point.Double(w1.(p224Wrapper).point)) +} + +func (w p224Wrapper) ScalarMult(w1 WrappedPoint, scalar []byte) (WrappedPoint, error) { + p, err := w.point.ScalarMult(w1.(p224Wrapper).point, scalar) + return wrapP224(p), err +} + +func (w p224Wrapper) ScalarBaseMult(scalar []byte) (WrappedPoint, error) { + p, err := w.point.ScalarBaseMult(scalar) + return wrapP224(p), err +} + +type p256Wrapper struct { + point *P256Point +} + +func wrapP256(point *P256Point) WrappedPoint { + return p256Wrapper{point: point} +} + +func NewP256WrappedPoint() WrappedPoint { + return wrapP256(NewP256Point()) +} + +func NewP256WrappedGenerator() WrappedPoint { + return wrapP256(NewP256Generator()) +} + +func (w p256Wrapper) Bytes() []byte { + return w.point.Bytes() +} + +func (w p256Wrapper) SetBytes(b []byte) (WrappedPoint, error) { + p, err := w.point.SetBytes(b) + return wrapP256(p), err +} + +func (w p256Wrapper) Add(w1, w2 WrappedPoint) WrappedPoint { + return wrapP256(w.point.Add(w1.(p256Wrapper).point, w2.(p256Wrapper).point)) +} + +func (w p256Wrapper) Double(w1 WrappedPoint) WrappedPoint { + return wrapP256(w.point.Double(w1.(p256Wrapper).point)) +} + +func (w p256Wrapper) ScalarMult(w1 WrappedPoint, scalar []byte) (WrappedPoint, error) { + p, err := w.point.ScalarMult(w1.(p256Wrapper).point, scalar) + return wrapP256(p), err +} + +func (w p256Wrapper) ScalarBaseMult(scalar []byte) (WrappedPoint, error) { + p, err := w.point.ScalarBaseMult(scalar) + return wrapP256(p), err +} + +type p521Wrapper struct { + point *P521Point +} + +func wrapP521(point *P521Point) WrappedPoint { + return p521Wrapper{point: point} +} + +func NewP521WrappedPoint() WrappedPoint { + return wrapP521(NewP521Point()) +} + +func NewP521WrappedGenerator() WrappedPoint { + return wrapP521(NewP521Generator()) +} + +func (w p521Wrapper) Bytes() []byte { + return w.point.Bytes() +} + +func (w p521Wrapper) SetBytes(b []byte) (WrappedPoint, error) { + p, err := w.point.SetBytes(b) + return wrapP521(p), err +} + +func (w p521Wrapper) Add(w1, w2 WrappedPoint) WrappedPoint { + return wrapP521(w.point.Add(w1.(p521Wrapper).point, w2.(p521Wrapper).point)) +} + +func (w p521Wrapper) Double(w1 WrappedPoint) WrappedPoint { + return wrapP521(w.point.Double(w1.(p521Wrapper).point)) +} + +func (w p521Wrapper) ScalarMult(w1 WrappedPoint, scalar []byte) (WrappedPoint, error) { + p, err := w.point.ScalarMult(w1.(p521Wrapper).point, scalar) + return wrapP521(p), err +} + +func (w p521Wrapper) ScalarBaseMult(scalar []byte) (WrappedPoint, error) { + p, err := w.point.ScalarBaseMult(scalar) + return wrapP521(p), err +} + +type p384Wrapper struct { + point *P384Point +} + +func wrapP384(point *P384Point) WrappedPoint { + return p384Wrapper{point: point} +} + +func NewP384WrappedPoint() WrappedPoint { + return wrapP384(NewP384Point()) +} + +func NewP384WrappedGenerator() WrappedPoint { + return wrapP384(NewP384Generator()) +} + +func (w p384Wrapper) Bytes() []byte { + return w.point.Bytes() +} + +func (w p384Wrapper) SetBytes(b []byte) (WrappedPoint, error) { + p, err := w.point.SetBytes(b) + return wrapP384(p), err +} + +func (w p384Wrapper) Add(w1, w2 WrappedPoint) WrappedPoint { + return wrapP384(w.point.Add(w1.(p384Wrapper).point, w2.(p384Wrapper).point)) +} + +func (w p384Wrapper) Double(w1 WrappedPoint) WrappedPoint { + return wrapP384(w.point.Double(w1.(p384Wrapper).point)) +} + +func (w p384Wrapper) ScalarMult(w1 WrappedPoint, scalar []byte) (WrappedPoint, error) { + p, err := w.point.ScalarMult(w1.(p384Wrapper).point, scalar) + return wrapP384(p), err +} + +func (w p384Wrapper) ScalarBaseMult(scalar []byte) (WrappedPoint, error) { + p, err := w.point.ScalarBaseMult(scalar) + return wrapP384(p), err +} diff --git a/compiler/natives/src/go/token/position.go b/compiler/natives/src/go/token/position.go new file mode 100644 index 000000000..c7fabb810 --- /dev/null +++ b/compiler/natives/src/go/token/position.go @@ -0,0 +1,26 @@ +//go:build js +// +build js + +package token + +import ( + "sync" + "sync/atomic" + "unsafe" +) + +type FileSet struct { + mutex sync.RWMutex + base int + files []*File + + // replaced atomic.Pointer[File] for go1.19 without generics + last atomicFilePointer +} + +type atomicFilePointer struct { + v unsafe.Pointer +} + +func (x *atomicFilePointer) Load() *File { return (*File)(atomic.LoadPointer(&x.v)) } +func (x *atomicFilePointer) Store(val *File) { atomic.StorePointer(&x.v, unsafe.Pointer(val)) } diff --git a/compiler/natives/src/hash/maphash/maphash.go b/compiler/natives/src/hash/maphash/maphash.go index 877366f04..dceff2c62 100644 --- a/compiler/natives/src/hash/maphash/maphash.go +++ b/compiler/natives/src/hash/maphash/maphash.go @@ -8,7 +8,7 @@ var hashkey [4]uint32 func init() { for i := range hashkey { - hashkey[i] = runtime_fastrand() + hashkey[i] = uint32(runtime_fastrand64()) } hashkey[0] |= 1 // make sure these numbers are odd hashkey[1] |= 1 diff --git a/compiler/natives/src/runtime/runtime.go b/compiler/natives/src/runtime/runtime.go index 7e76e3ccc..41c60876c 100644 --- a/compiler/natives/src/runtime/runtime.go +++ b/compiler/natives/src/runtime/runtime.go @@ -489,5 +489,6 @@ func throw(s string) { } func nanotime() int64 { - return js.Global.Get("Date").New().Call("getTime").Int64() * int64(1000_000) + const millisecond = 1_000_000 + return js.Global.Get("Date").New().Call("getTime").Int64() * millisecond } diff --git a/compiler/natives/src/sync/atomic/atomic.go b/compiler/natives/src/sync/atomic/atomic.go index ebc98e910..1cbfe65f9 100644 --- a/compiler/natives/src/sync/atomic/atomic.go +++ b/compiler/natives/src/sync/atomic/atomic.go @@ -220,3 +220,6 @@ func sameType(x, y interface{}) bool { // existing and differing for different types. return js.InternalObject(x).Get("constructor") == js.InternalObject(y).Get("constructor") } + +//gopherjs:purge for go1.19 without generics +type Pointer[T any] struct{} diff --git a/compiler/natives/src/sync/atomic/atomic_test.go b/compiler/natives/src/sync/atomic/atomic_test.go index f4450cc67..27ce36df9 100644 --- a/compiler/natives/src/sync/atomic/atomic_test.go +++ b/compiler/natives/src/sync/atomic/atomic_test.go @@ -3,7 +3,51 @@ package atomic_test -import "testing" +import ( + "testing" + "unsafe" +) + +//gopherjs:purge for go1.19 without generics +func testPointers() []unsafe.Pointer {} + +func TestSwapPointer(t *testing.T) { + t.Skip("GopherJS does not support generics yet.") +} + +func TestSwapPointerMethod(t *testing.T) { + t.Skip("GopherJS does not support generics yet.") +} + +func TestCompareAndSwapPointer(t *testing.T) { + t.Skip("GopherJS does not support generics yet.") +} + +func TestCompareAndSwapPointerMethod(t *testing.T) { + t.Skip("GopherJS does not support generics yet.") +} + +func TestLoadPointer(t *testing.T) { + t.Skip("GopherJS does not support generics yet.") +} + +func TestLoadPointerMethod(t *testing.T) { + t.Skip("GopherJS does not support generics yet.") +} + +func TestStorePointer(t *testing.T) { + t.Skip("GopherJS does not support generics yet.") +} + +func TestStorePointerMethod(t *testing.T) { + t.Skip("GopherJS does not support generics yet.") +} + +//gopherjs:purge for go1.19 without generics +func hammerStoreLoadPointer(t *testing.T, paddr unsafe.Pointer) {} + +//gopherjs:purge for go1.19 without generics +func hammerStoreLoadPointerMethod(t *testing.T, paddr unsafe.Pointer) {} func TestHammerStoreLoad(t *testing.T) { t.Skip("use of unsafe") @@ -12,3 +56,10 @@ func TestHammerStoreLoad(t *testing.T) { func TestUnaligned64(t *testing.T) { t.Skip("GopherJS emulates atomics, which makes alignment irrelevant.") } + +func TestNilDeref(t *testing.T) { + t.Skip("GopherJS does not support generics yet.") +} + +//gopherjs:purge for go1.19 without generics +type List struct{} diff --git a/compiler/natives/src/sync/sync.go b/compiler/natives/src/sync/sync.go index 588537751..294b0b109 100644 --- a/compiler/natives/src/sync/sync.go +++ b/compiler/natives/src/sync/sync.go @@ -3,7 +3,11 @@ package sync -import "github.com/gopherjs/gopherjs/js" +import ( + _ "unsafe" // For go:linkname + + "github.com/gopherjs/gopherjs/js" +) var semWaiters = make(map[*uint32][]chan bool) @@ -69,11 +73,8 @@ func runtime_canSpin(i int) bool { return false } -// Copy of time.runtimeNano. -func runtime_nanotime() int64 { - const millisecond = 1000000 - return js.Global.Get("Date").New().Call("getTime").Int64() * millisecond -} +//go:linkname runtime_nanotime runtime.nanotime +func runtime_nanotime() int64 // Implemented in runtime. func throw(s string) { diff --git a/compiler/natives/src/testing/helper_test.go b/compiler/natives/src/testing/helper_test.go index b277fa31f..6815fd651 100644 --- a/compiler/natives/src/testing/helper_test.go +++ b/compiler/natives/src/testing/helper_test.go @@ -2,3 +2,7 @@ // +build js package testing + +func TestTBHelper(t *T) { + t.Skip("GopherJS does not support generics yet.") +} diff --git a/compiler/natives/src/testing/helperfuncs_test.go b/compiler/natives/src/testing/helperfuncs_test.go new file mode 100644 index 000000000..54a1ee737 --- /dev/null +++ b/compiler/natives/src/testing/helperfuncs_test.go @@ -0,0 +1,13 @@ +//go:build js +// +build js + +package testing + +//gopherjs:purge for go1.19 without generics +func genericHelper[G any](t *T, msg string) + +//gopherjs:purge for go1.19 without generics +var genericIntHelper = genericHelper[int] + +//gopherjs:purge for go1.19 without generics (uses genericHelper) +func testHelper(t *T) From 5a980432222d942e19b414eff53e665560b94ad7 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Fri, 19 Jan 2024 11:39:12 -0700 Subject: [PATCH 13/29] Updating documentation for build directives --- build/build.go | 14 +++- doc/pargma.md | 174 ++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 176 insertions(+), 12 deletions(-) diff --git a/build/build.go b/build/build.go index 30e2b15aa..a6832a607 100644 --- a/build/build.go +++ b/build/build.go @@ -150,11 +150,16 @@ type overrideInfo struct { // - For function identifiers that exist in the original and the overrides // and have the directive `gopherjs:keep-original`, the original identifier // in the AST gets prefixed by `_gopherjs_original_`. -// - For identifiers that exist in the original and the overrides and have +// - For identifiers that exist in the original and the overrides, and have // the directive `gopherjs:purge`, both the original and override are // removed. This is for completely removing something which is currently // invalid for GopherJS. For any purged types any methods with that type as // the receiver are also removed. +// - For function identifiers that exist in the original and the overrides, +// and have the directive `gopherjs:override-signature`, the overridden +// function is removed and the original function's signature is changed +// to match the overridden function signature. This allows the receiver, +// type parameters, parameter, and return values to be modified as needed. // - Otherwise for identifiers that exist in the original and the overrides, // the original is removed. // - New identifiers that don't exist in original package get added. @@ -414,9 +419,12 @@ func augmentOriginalFile(file *ast.File, overrides map[string]overrideInfo) { // isOnlyImports determines if this file is empty except for imports. func isOnlyImports(file *ast.File) bool { for _, decl := range file.Decls { - if gen, ok := decl.(*ast.GenDecl); !ok || gen.Tok != token.IMPORT { - return false + if gen, ok := decl.(*ast.GenDecl); ok && gen.Tok == token.IMPORT { + continue } + + // The decl was either a FuncDecl or a non-import GenDecl. + return false } return true } diff --git a/doc/pargma.md b/doc/pargma.md index a13c64ce6..2bcf71f93 100644 --- a/doc/pargma.md +++ b/doc/pargma.md @@ -7,6 +7,12 @@ issues, so it is recommended to avoid using them if possible. GopherJS compiler supports the following directives: +- [go:linkname](#golinkname) +- [go:embed](#goembed) +- [gopherjs:keep-original](#gopherjskeep-original) +- [gopherjs:purge](#gopherjspurge) +- [gopherjs:override-signature](#gopherjsoverride-signature) + ## `go:linkname` This is a limited version of the `go:linkname` directive the upstream Go @@ -25,16 +31,166 @@ Signatures of `remotename` and `localname` must be identical. Since this directive can subvert package incapsulation, the source file that uses the directive must also import `unsafe`. -The following directive format is supported: -//go:linkname . -//go:linkname .. -//go:linkname .<(*type)>. +The following directive formats are supported: + +- `//go:linkname .` +- `//go:linkname ..` +- `//go:linkname .<(*type)>.` Compared to the upstream Go, the following limitations exist in GopherJS: - - The directive only works on package-level functions or methods (variables - are not supported). - - The directive can only be used to "import" implementation from another - package, and not to "provide" local implementation to another package. +- The directive only works on package-level functions or methods (variables + are not supported). +- The directive can only be used to "import" implementation from another + package, and not to "provide" local implementation to another package. + +See [gopherjs/issues/1000](https://github.com/gopherjs/gopherjs/issues/1000) +for details. + +## `go:embed` + +This is a very similar version of the `go:embed` directive the upstream Go +compiler implements. +GopherJS leverages [goembed](https://github.com/visualfc/goembed) +to parse this directive and provide support reading embedded content. Usage: + +```go +import _ "embed" // for go:embed + +//go:embed externalText +var embeddedText string + +//go:embed externalContent +var embeddedContent []byte + +//go:embed file1 +//go:embed file2 +// ... +//go:embed image/* blobs/* +var embeddedFiles embed.FS +``` + +This directive affects the variable specification (e.g. `embeddedText`) +that the comment containing the directive is associated with. +There may be one embed directives associated with `string` or `[]byte` +variables. There may be one or more embed directives associated with +`embed.FS` variables and each directive may contain one or more +file matching patterns. The effect is that the variable will be assigned to +the content (e.g. `externalText`) given in the directive. In the case +of `embed.FS`, several embedded files will be accessible. + +See [pkg.go.dev/embed](https://pkg.go.dev/embed#hdr-Directives) +for more information. + +## `gopherjs:keep-original` + +This directive is custom to GopherJS. This directive can be added to a +function declaration in the native file overrides as part of the build step. + +This will keep the original function by the same name as the function +in the overrides, however it will prepend `_gopherjs_original_` to the original +function's name. This allows the original function to be called by functions +in the overrides and the overridden function to be called instead of the +original. This is useful when wanting to augment the original behavior without +having to rewrite the entire original function. Usage: + +```go +//gopherjs:keep-original +func foo(a, b int) int { + return _gopherjs_original_foo(a+1, b+1) - 1 +} +``` + +## `gopherjs:purge` + +This directive is custom to GopherJS. This directive can be added +to most declarations and specification in the native file overrides as +part of the build step. +This can be added to structures, interfaces, methods, functions, +variables, or constants, but are not supported for imports, structure fields, +nor interface function signatures. -See https://github.com/gopherjs/gopherjs/issues/1000 for details. +This will remove the original structure, interface, etc from both the override +files and the original files. +If this is added to a structure, then all functions in the original files +that use that structure as a receiver will also be removed. +This is useful for removing all the code that is invalid in GopherJS, +such as code using unsupported features (e.g. generic interfaces before +generics were fully supported). In many cases the overrides to replace +the original code may not have use of all the original functions and +variables or the original code is not intended to be replaced yet. +Usage: + +```go +//gopherjs:purge +var data string + +//gopherjs:purge +// This will also purge any function starting with `dataType` as the receiver. +type dataType struct {} + +//gopherjs:purge +type interfaceType interface{} + +//gopherjs:purge +func doThing[T ~string](value T) +``` + +## `gopherjs:override-signature` + +This directive is custom to GopherJS. This directive can be added to a +function declaration in the native file overrides as part of the build step. + +This will remove the function from the overrides but record the signature +used in the overrides, then update the original function with that signature +provided in the overrides. +The affect is to change the receiver, type parameters, +parameters, or return types of the original function. The original function +and override function must have the same function key name so that they can +be associated, meaning the identifier of the receiver, if there is one, must +match and the identifier of the function must match. + +This allows the signature to be modified without modifying the body of a +function thus allowing the types to be adjusted to work in GopherJS. +The signature may need to be replaced because it uses a parameter type +that is invalid in GopherJS or the signature uses unsupported features +(e.g. generic interfaces before generics were fully supported). +Usage: + +```go +// -- in original file -- +func Foo[T comparable](a, b T) (T, bool) { + if a == b { + return a, true + } + return b, false +} + +// -- in override file -- +//gopherjs:override-signature +func Foo(a, b any) (any, bool) + +// -- result in augmented original -- +func Foo(a, b any) (any, bool) { + if a == b { + return a, true + } + return b, false +} +``` + +```go +// -- in original file -- +func (f *Foo[A, B, C]) Bar(a int, b *A) (*A, error) { + //... +} + +// -- in override file -- +//gopherjs:override-signature +func (f *Foo) Bar(a int, b jsTypeA) (jsTypeA, error) + +// -- result in augmented original -- +func (f *Foo) Bar(a int, b jsTypeA) (jsTypeA, error) { + //... +} +``` From 12a247d9a8bfdd5cdc8bf10feb88d70b7b3e4f14 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 30 Jan 2024 16:26:14 -0700 Subject: [PATCH 14/29] Fix problem where import is named but source is augmented to only need import for a directive --- build/build.go | 2 ++ build/build_test.go | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/build/build.go b/build/build.go index a6832a607..d3cd32952 100644 --- a/build/build.go +++ b/build/build.go @@ -475,6 +475,8 @@ func pruneImports(file *ast.File) { path, _ := strconv.Unquote(in.Path.Value) directivePrefix, hasPath := directiveImports[path] if hasPath && astutil.HasDirectivePrefix(file, directivePrefix) { + // since the import is otherwise unused set the name to blank. + in.Name = ast.NewIdent(`_`) delete(unused, name) if len(unused) == 0 { return diff --git a/build/build_test.go b/build/build_test.go index f54281cb8..5f025540b 100644 --- a/build/build_test.go +++ b/build/build_test.go @@ -396,7 +396,14 @@ func TestOverlayAugmentation(t *testing.T) { //go:linkname runtimeNano runtime.nanotime func runtimeNano() int64`, - noCodeChange: true, + want: `import _ "unsafe" + import "embed" + + //go:embed hello.txt + var eFile embed.FS + + //go:linkname runtimeNano runtime.nanotime + func runtimeNano() int64`, expInfo: map[string]overrideInfo{ `eFile`: {}, `runtimeNano`: {}, From 4b5b0771d02d2db3484258727e59c151284034f5 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 30 Jan 2024 16:27:48 -0700 Subject: [PATCH 15/29] Fix chocolatey go version --- circle.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 62031ce4b..5c059a6e4 100644 --- a/circle.yml +++ b/circle.yml @@ -55,6 +55,10 @@ parameters: go_version: type: string default: "1.19.13" + chocolatey_go_version: + type: string + # Chocolatey doesn't have 1.19.13, closest is 1.19.9 + default: "1.19.9" nvm_version: type: string default: "0.38.0" @@ -171,7 +175,7 @@ jobs: - run: name: Install Go command: | - choco install golang --version="<< pipeline.parameters.go_version >>" -my + choco install golang --version="<< pipeline.parameters.chocolatey_go_version >>" -my go version (Get-Command go).Path [Environment]::SetEnvironmentVariable( From d7abc77ddbbc45ec6aa20a8f10c8840de5f1017b Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 30 Jan 2024 16:30:19 -0700 Subject: [PATCH 16/29] Update known fails list of fixed bugs --- tests/gorepo/run.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/gorepo/run.go b/tests/gorepo/run.go index 0ceb99ff5..d58968ada 100644 --- a/tests/gorepo/run.go +++ b/tests/gorepo/run.go @@ -151,10 +151,16 @@ var knownFails = map[string]failReason{ // These are new tests in Go 1.18 "fixedbugs/issue46938.go": {category: notApplicable, desc: "tests -d=checkptr compiler mode, which GopherJS doesn't support"}, "fixedbugs/issue47928.go": {category: notApplicable, desc: "//go:nointerface is a part of GOEXPERIMENT=fieldtrack and is not supported by GopherJS"}, - "fixedbugs/issue49665.go": {category: other, desc: "attempts to pass -gcflags=-G=3 to enable generics, GopherJS doesn't expect the flag; re-enable in Go 1.19 where the flag is removed"}, "fixedbugs/issue48898.go": {category: other, desc: "https://github.com/gopherjs/gopherjs/issues/1128"}, "fixedbugs/issue48536.go": {category: usesUnsupportedPackage, desc: "https://github.com/gopherjs/gopherjs/issues/1130"}, "fixedbugs/issue53600.go": {category: lowLevelRuntimeDifference, desc: "GopherJS println format is different from Go's"}, + + // These are new tests in Go 1.19 + "fixedbugs/issue50672.go": {category: usesUnsupportedGenerics, desc: "Checking function nesting with one function having a type parameter."}, + "fixedbugs/issue53137.go": {category: usesUnsupportedGenerics, desc: "Checking setting type parameter of struct in parameter of a generic function."}, + "fixedbugs/issue53309.go": {category: usesUnsupportedGenerics, desc: "Checking unused type parameter in method call to interface"}, + "fixedbugs/issue53635.go": {category: usesUnsupportedGenerics, desc: "Checking switch type against nil type with unsupported type parameters"}, + "fixedbugs/issue53653.go": {category: lowLevelRuntimeDifference, desc: "GopherJS println format of int64 is different from Go's"}, } type failCategory uint8 @@ -164,6 +170,7 @@ const ( neverTerminates // Test never terminates (so avoid starting it). usesUnsupportedPackage // Test fails because it imports an unsupported package, e.g., "unsafe". requiresSourceMapSupport // Test fails without source map support (as configured in CI), because it tries to check filename/line number via runtime.Caller. + usesUnsupportedGenerics // Test uses generics (type parameters) that are not currently supported. compilerPanic unsureIfGopherJSSupportsThisFeature lowLevelRuntimeDifference // JavaScript runtime behaves differently from Go in ways that are difficult to work around. From 91f1d44246b326ba7bf3d7022ed09a34ea3e7da3 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 30 Jan 2024 17:10:29 -0700 Subject: [PATCH 17/29] Updated crypto natives --- .../crypto/internal/boring/bcache/cache.go | 30 +++++++++++++++++++ .../internal/boring/bcache/cache_test.go | 10 +++++++ .../src/crypto/internal/boring/sig/sig.go | 13 ++++++++ .../src/crypto/internal/nistec/nistec_test.go | 18 ++++++----- 4 files changed, 63 insertions(+), 8 deletions(-) create mode 100644 compiler/natives/src/crypto/internal/boring/bcache/cache.go create mode 100644 compiler/natives/src/crypto/internal/boring/bcache/cache_test.go create mode 100644 compiler/natives/src/crypto/internal/boring/sig/sig.go diff --git a/compiler/natives/src/crypto/internal/boring/bcache/cache.go b/compiler/natives/src/crypto/internal/boring/bcache/cache.go new file mode 100644 index 000000000..afff404ce --- /dev/null +++ b/compiler/natives/src/crypto/internal/boring/bcache/cache.go @@ -0,0 +1,30 @@ +//go:build js +// +build js + +package bcache + +import "unsafe" + +// Cache relies on GC to periodically clear the cache. +// Since GopherJS doesn't have the same GC hooks, it currently can not +// register this cache with the GC. +// Without this cache Boring crypto, in particular public and private +// RSA and ECDSA keys, will be slower because the cache will always miss. +type Cache struct{} + +func (c *Cache) Register() {} +func (c *Cache) Clear() {} +func (c *Cache) Get(k unsafe.Pointer) unsafe.Pointer { return nil } +func (c *Cache) Put(k, v unsafe.Pointer) {} + +//gopherjs:purge +func (c *Cache) table() *[cacheSize]unsafe.Pointer + +//gopherjs:purge +type cacheEntry struct{} + +//gopherjs:purge +func registerCache(unsafe.Pointer) + +//gopherjs:purge +const cacheSize = 1021 diff --git a/compiler/natives/src/crypto/internal/boring/bcache/cache_test.go b/compiler/natives/src/crypto/internal/boring/bcache/cache_test.go new file mode 100644 index 000000000..12f2c4da4 --- /dev/null +++ b/compiler/natives/src/crypto/internal/boring/bcache/cache_test.go @@ -0,0 +1,10 @@ +//go:build js +// +build js + +package bcache + +import "testing" + +func TestCache(t *testing.T) { + t.Skip(`This test uses runtime.GC(), which GopherJS doesn't support`) +} diff --git a/compiler/natives/src/crypto/internal/boring/sig/sig.go b/compiler/natives/src/crypto/internal/boring/sig/sig.go new file mode 100644 index 000000000..3eb2454aa --- /dev/null +++ b/compiler/natives/src/crypto/internal/boring/sig/sig.go @@ -0,0 +1,13 @@ +//go:build js +// +build js + +package sig + +// Setting to no-op +func BoringCrypto() {} + +// Setting to no-op +func FIPSOnly() {} + +// Setting to no-op +func StandardCrypto() {} diff --git a/compiler/natives/src/crypto/internal/nistec/nistec_test.go b/compiler/natives/src/crypto/internal/nistec/nistec_test.go index f89aaeae0..f5f79398e 100644 --- a/compiler/natives/src/crypto/internal/nistec/nistec_test.go +++ b/compiler/natives/src/crypto/internal/nistec/nistec_test.go @@ -4,8 +4,10 @@ package nistec_test import ( - "crypto/elliptic" "testing" + + "crypto/elliptic" + "crypto/internal/nistec" ) func TestAllocations(t *testing.T) { @@ -31,7 +33,7 @@ func TestEquivalents(t *testing.T) { } //gopherjs:override-signature -func testEquivalents(t *testing.T, newPoint, newGenerator func() WrappedPoint, c elliptic.Curve) {} +func testEquivalents(t *testing.T, newPoint, newGenerator func() nistec.WrappedPoint, c elliptic.Curve) func TestScalarMult(t *testing.T) { t.Run("P224", func(t *testing.T) { @@ -49,14 +51,14 @@ func TestScalarMult(t *testing.T) { } //gopherjs:override-signature -func testScalarMult(t *testing.T, newPoint, newGenerator func() WrappedPoint, c elliptic.Curve) +func testScalarMult(t *testing.T, newPoint, newGenerator func() nistec.WrappedPoint, c elliptic.Curve) func BenchmarkScalarMult(b *testing.B) { b.Run("P224", func(b *testing.B) { benchmarkScalarMult(b, nistec.NewP224WrappedGenerator(), 28) }) b.Run("P256", func(b *testing.B) { - benchmarkScalarMult(b, nistec.NewP256GWrappedenerator(), 32) + benchmarkScalarMult(b, nistec.NewP256WrappedGenerator(), 32) }) b.Run("P384", func(b *testing.B) { benchmarkScalarMult(b, nistec.NewP384WrappedGenerator(), 48) @@ -67,11 +69,11 @@ func BenchmarkScalarMult(b *testing.B) { } //gopherjs:override-signature -func benchmarkScalarMult(b *testing.B, p WrappedPoint, scalarSize int) +func benchmarkScalarMult(b *testing.B, p nistec.WrappedPoint, scalarSize int) func BenchmarkScalarBaseMult(b *testing.B) { b.Run("P224", func(b *testing.B) { - benchmarkScalarBaseMult(b, nistec.NewP22Wrapped4Generator(), 28) + benchmarkScalarBaseMult(b, nistec.NewP224WrappedGenerator(), 28) }) b.Run("P256", func(b *testing.B) { benchmarkScalarBaseMult(b, nistec.NewP256WrappedGenerator(), 32) @@ -80,9 +82,9 @@ func BenchmarkScalarBaseMult(b *testing.B) { benchmarkScalarBaseMult(b, nistec.NewP384WrappedGenerator(), 48) }) b.Run("P521", func(b *testing.B) { - benchmarkScalarBaseMult(b, nistec.NewP521GWrappedenerator(), 66) + benchmarkScalarBaseMult(b, nistec.NewP521WrappedGenerator(), 66) }) } //gopherjs:override-signature -func benchmarkScalarBaseMult(b *testing.B, p WrappedPoint, scalarSize int) +func benchmarkScalarBaseMult(b *testing.B, p nistec.WrappedPoint, scalarSize int) From f5c911a46f9dd13f8cadb00e789a87a852dca703 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 30 Jan 2024 17:10:59 -0700 Subject: [PATCH 18/29] Fix natives for debug/pe --- compiler/natives/src/debug/pe/symbol.go | 119 ++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 compiler/natives/src/debug/pe/symbol.go diff --git a/compiler/natives/src/debug/pe/symbol.go b/compiler/natives/src/debug/pe/symbol.go new file mode 100644 index 000000000..798502ce3 --- /dev/null +++ b/compiler/natives/src/debug/pe/symbol.go @@ -0,0 +1,119 @@ +//go:build js +// +build js + +package pe + +import ( + "encoding/binary" + "fmt" + "io" +) + +// bytesBufferLite is a simplified bytes.Buffer to avoid +// including `bytes` as a new import into the pe package. +type bytesBufferLite struct { + data []byte + off int +} + +func (buf *bytesBufferLite) Write(p []byte) (int, error) { + buf.data = append(buf.data, p...) + return len(p), nil +} + +func (buf *bytesBufferLite) Read(p []byte) (int, error) { + n := copy(p, buf.data[buf.off:]) + buf.off += n + return n, nil +} + +func copyToAuxFormat5(sym *COFFSymbol) (*COFFSymbolAuxFormat5, error) { + buf := &bytesBufferLite{data: make([]byte, 0, 20)} + if err := binary.Write(buf, binary.LittleEndian, sym); err != nil { + return nil, err + } + aux := &COFFSymbolAuxFormat5{} + if err := binary.Read(buf, binary.LittleEndian, aux); err != nil { + return nil, err + } + return aux, nil +} + +func copyFromAuxFormat5(aux *COFFSymbolAuxFormat5) (*COFFSymbol, error) { + buf := &bytesBufferLite{data: make([]byte, 0, 20)} + if err := binary.Write(buf, binary.LittleEndian, aux); err != nil { + return nil, err + } + sym := &COFFSymbol{} + if err := binary.Read(buf, binary.LittleEndian, sym); err != nil { + return nil, err + } + return sym, nil +} + +func readCOFFSymbols(fh *FileHeader, r io.ReadSeeker) ([]COFFSymbol, error) { + if fh.PointerToSymbolTable == 0 { + return nil, nil + } + if fh.NumberOfSymbols <= 0 { + return nil, nil + } + _, err := r.Seek(int64(fh.PointerToSymbolTable), seekStart) + if err != nil { + return nil, fmt.Errorf("fail to seek to symbol table: %v", err) + } + syms := make([]COFFSymbol, fh.NumberOfSymbols) + naux := 0 + for k := range syms { + if naux == 0 { + err = binary.Read(r, binary.LittleEndian, &syms[k]) + if err != nil { + return nil, fmt.Errorf("fail to read symbol table: %v", err) + } + naux = int(syms[k].NumberOfAuxSymbols) + } else { + naux-- + // The following was reading into one struct with the same memory + // footprint as another struck. This doesn't work in JS so the + // `syms` value is left with a bunch of defaults. So replace + // aux := (*COFFSymbolAuxFormat5)(unsafe.Pointer(&syms[k])) + // (an in memory remap) with the following read and then copy. + aux := &COFFSymbolAuxFormat5{} + err = binary.Read(r, binary.LittleEndian, aux) + if err != nil { + return nil, fmt.Errorf("fail to read symbol table: %v", err) + } + pesymn, err := copyFromAuxFormat5(aux) + if err != nil { + return nil, err + } + syms[k] = *pesymn + } + } + if naux != 0 { + return nil, fmt.Errorf("fail to read symbol table: %d aux symbols unread", naux) + } + return syms, nil +} + +func (f *File) COFFSymbolReadSectionDefAux(idx int) (*COFFSymbolAuxFormat5, error) { + var rv *COFFSymbolAuxFormat5 + if idx < 0 || idx >= len(f.COFFSymbols) { + return rv, fmt.Errorf("invalid symbol index") + } + pesym := &f.COFFSymbols[idx] + const IMAGE_SYM_CLASS_STATIC = 3 + if pesym.StorageClass != uint8(IMAGE_SYM_CLASS_STATIC) { + return rv, fmt.Errorf("incorrect symbol storage class") + } + if pesym.NumberOfAuxSymbols == 0 || idx+1 >= len(f.COFFSymbols) { + return rv, fmt.Errorf("aux symbol unavailable") + } + pesymn := &f.COFFSymbols[idx+1] + // The following was reading one struct as another struct with + // the same memory footprint. This doesn't work in JS so the + // `rv` value is left with a bunch of `undefined`s. So replace + // rv = (*COFFSymbolAuxFormat5)(unsafe.Pointer(pesymn)) + // (an in memory remap) with the following copy. + return copyToAuxFormat5(pesymn) +} From ff493e49e8e15eda498acb4f83216e77b3c00c5d Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 30 Jan 2024 17:16:30 -0700 Subject: [PATCH 19/29] Update to go/token/position --- compiler/natives/src/go/token/position.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/compiler/natives/src/go/token/position.go b/compiler/natives/src/go/token/position.go index c7fabb810..6a1ee0c15 100644 --- a/compiler/natives/src/go/token/position.go +++ b/compiler/natives/src/go/token/position.go @@ -3,24 +3,20 @@ package token -import ( - "sync" - "sync/atomic" - "unsafe" -) +import "sync" type FileSet struct { mutex sync.RWMutex base int files []*File - // replaced atomic.Pointer[File] for go1.19 without generics + // replaced atomic.Pointer[File] for go1.19 without generics. last atomicFilePointer } type atomicFilePointer struct { - v unsafe.Pointer + v *File } -func (x *atomicFilePointer) Load() *File { return (*File)(atomic.LoadPointer(&x.v)) } -func (x *atomicFilePointer) Store(val *File) { atomic.StorePointer(&x.v, unsafe.Pointer(val)) } +func (x *atomicFilePointer) Load() *File { return x.v } +func (x *atomicFilePointer) Store(val *File) { x.v = val } From cd055c9e5a7918e7a94de6871e6713d82ef51652 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 30 Jan 2024 17:22:51 -0700 Subject: [PATCH 20/29] Update hash/maphash --- compiler/natives/src/hash/maphash/maphash.go | 96 ++++++++++++++------ 1 file changed, 66 insertions(+), 30 deletions(-) diff --git a/compiler/natives/src/hash/maphash/maphash.go b/compiler/natives/src/hash/maphash/maphash.go index dceff2c62..5c982404f 100644 --- a/compiler/natives/src/hash/maphash/maphash.go +++ b/compiler/natives/src/hash/maphash/maphash.go @@ -3,31 +3,58 @@ package maphash -// used in hash{32,64}.go to seed the hash function -var hashkey [4]uint32 +import ( + _ "unsafe" // for linkname +) + +// hashkey is similar how it is defined in runtime/alg.go for Go 1.19 +// to be used in hash{32,64}.go to seed the hash function as part of +// runtime_memhash. We're using locally defined memhash so it got moved here. +var hashkey [3]uint32 func init() { for i := range hashkey { - hashkey[i] = uint32(runtime_fastrand64()) + hashkey[i] = runtime_fastrand() | 1 + // The `| 1` is to make sure these numbers are odd } - hashkey[0] |= 1 // make sure these numbers are odd - hashkey[1] |= 1 - hashkey[2] |= 1 - hashkey[3] |= 1 } -func _rthash(b []byte, seed uint64) uint64 { +//go:linkname runtime_fastrand runtime.fastrand +func runtime_fastrand() uint32 + +// Bytes uses less efficient equivalent to avoid using unsafe. +func Bytes(seed Seed, b []byte) uint64 { + var h Hash + h.SetSeed(seed) + _, _ = h.Write(b) + return h.Sum64() +} + +// String uses less efficient equivalent to avoid using unsafe. +func String(seed Seed, s string) uint64 { + var h Hash + h.SetSeed(seed) + _, _ = h.WriteString(s) + return h.Sum64() +} + +// rthash is similar to the Go 1.19.13 version +// with the call to memhash changed to not use unsafe pointers. +func rthash(b []byte, seed uint64) uint64 { if len(b) == 0 { return seed } // The runtime hasher only works on uintptr. Since GopherJS implements a // 32-bit environment, we use two parallel hashers on the lower and upper 32 // bits. - lo := memhash(b, uint32(seed), uint32(len(b))) - hi := memhash(b, uint32(seed>>32), uint32(len(b))) + lo := memhash(b, uint32(seed)) + hi := memhash(b, uint32(seed>>32)) return uint64(hi)<<32 | uint64(lo) } +//gopherjs:purge to remove link using unsafe pointers, use memhash instead. +func runtime_memhash() + // The implementation below is adapted from the upstream runtime/hash32.go // and avoids use of unsafe, which GopherJS doesn't support well and leads to // worse performance. @@ -38,8 +65,9 @@ func _rthash(b []byte, seed uint64) uint64 { // // Hashing algorithm inspired by wyhash: // https://github.com/wangyi-fudan/wyhash/blob/ceb019b530e2c1c14d70b79bfa2bc49de7d95bc1/Modern%20Non-Cryptographic%20Hash%20Function%20and%20Pseudorandom%20Number%20Generator.pdf -func memhash(p []byte, seed uint32, s uint32) uintptr { - a, b := mix32(uint32(seed), uint32(s^hashkey[0])) +func memhash(p []byte, seed uint32) uintptr { + s := len(p) + a, b := mix32(uint32(seed), uint32(s)^hashkey[0]) if s == 0 { return uintptr(a ^ b) } @@ -63,7 +91,7 @@ func memhash(p []byte, seed uint32, s uint32) uintptr { return uintptr(a ^ b) } -func add(p []byte, x uint32) []byte { +func add(p []byte, x int) []byte { return p[x:] } @@ -80,51 +108,59 @@ func mix32(a, b uint32) (uint32, uint32) { /* The following functions were modified in Go 1.17 to improve performance, but at the expense of being unsafe, and thus incompatible with GopherJS. - To compensate, we have reverted these to the unoptimized Go 1.16 versions - for now. + See https://cs.opensource.google/go/go/+/refs/tags/go1.19.13:src/hash/maphash/maphash.go; + To compensate, we use a simplified version of each method from Go 1.19.13, + similar to Go 1.16's versions, with the call to rthash changed to not use unsafe pointers. See upstream issue https://github.com/golang/go/issues/47342 to implement a purego version of this package, which should render this hack (and likely this entire file) obsolete. */ -// Write is borrowed from Go 1.16. +// Write is a simplification from Go 1.19 changed to not use unsafe. func (h *Hash) Write(b []byte) (int, error) { size := len(b) - for h.n+len(b) > len(h.buf) { - k := copy(h.buf[h.n:], b) - h.n = len(h.buf) - b = b[k:] - h.flush() + if h.n+len(b) > bufSize { + h.initSeed() + for h.n+len(b) > bufSize { + k := copy(h.buf[h.n:], b) + h.state.s = rthash(h.buf[:], h.state.s) + b = b[k:] + h.n = 0 + } } h.n += copy(h.buf[h.n:], b) return size, nil } -// WriteString is borrowed from Go 1.16. +// WriteString is a simplification from Go 1.19 changed to not use unsafe. func (h *Hash) WriteString(s string) (int, error) { size := len(s) - for h.n+len(s) > len(h.buf) { - k := copy(h.buf[h.n:], s) - h.n = len(h.buf) - s = s[k:] - h.flush() + if h.n+len(s) > bufSize { + h.initSeed() + for h.n+len(s) > bufSize { + k := copy(h.buf[h.n:], s) + h.state.s = rthash(h.buf[:], h.state.s) + s = s[k:] + h.n = 0 + } } h.n += copy(h.buf[h.n:], s) return size, nil } +// flush is the Go 1.19 version changed to not use unsafe. func (h *Hash) flush() { if h.n != len(h.buf) { panic("maphash: flush of partially full buffer") } h.initSeed() - h.state.s = _rthash(h.buf[:], h.state.s) + h.state.s = rthash(h.buf[:], h.state.s) h.n = 0 } -// Sum64 is borrowed from Go 1.16. +// Sum64 is the Go 1.19 version changed to not use unsafe. func (h *Hash) Sum64() uint64 { h.initSeed() - return _rthash(h.buf[:h.n], h.state.s) + return rthash(h.buf[:h.n], h.state.s) } From 543764212241adf2132f8c7c9523e83b477b70c8 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 30 Jan 2024 17:30:54 -0700 Subject: [PATCH 21/29] Update to reflect and reflectlite --- .../src/internal/reflectlite/all_test.go | 24 +++++ compiler/natives/src/reflect/reflect.go | 91 +++++++++++++++---- compiler/natives/src/reflect/reflect_test.go | 11 ++- 3 files changed, 104 insertions(+), 22 deletions(-) diff --git a/compiler/natives/src/internal/reflectlite/all_test.go b/compiler/natives/src/internal/reflectlite/all_test.go index 977438e4e..4445189a0 100644 --- a/compiler/natives/src/internal/reflectlite/all_test.go +++ b/compiler/natives/src/internal/reflectlite/all_test.go @@ -21,3 +21,27 @@ func TestTypes(t *testing.T) { func TestNameBytesAreAligned(t *testing.T) { t.Skip("TestNameBytesAreAligned") } + +// `A` is used with `B[T any]` and is otherwise not needed. +// +//gopherjs:purge for go1.19 without generics +type ( + A struct{} + B[T any] struct{} +) + +// removing the name tests using `B[T any]` for go1.19 without generics +var nameTests = []nameTest{ + {(*int32)(nil), "int32"}, + {(*D1)(nil), "D1"}, + {(*[]D1)(nil), ""}, + {(*chan D1)(nil), ""}, + {(*func() D1)(nil), ""}, + {(*<-chan D1)(nil), ""}, + {(*chan<- D1)(nil), ""}, + {(*any)(nil), ""}, + {(*interface { + F() + })(nil), ""}, + {(*TheNameOfThisTypeIsExactly255BytesLongSoWhenTheCompilerPrependsTheReflectTestPackageNameAndExtraStarTheLinkerRuntimeAndReflectPackagesWillHaveToCorrectlyDecodeTheSecondLengthByte0123456789_0123456789_0123456789_0123456789_0123456789_012345678)(nil), "TheNameOfThisTypeIsExactly255BytesLongSoWhenTheCompilerPrependsTheReflectTestPackageNameAndExtraStarTheLinkerRuntimeAndReflectPackagesWillHaveToCorrectlyDecodeTheSecondLengthByte0123456789_0123456789_0123456789_0123456789_0123456789_012345678"}, +} diff --git a/compiler/natives/src/reflect/reflect.go b/compiler/natives/src/reflect/reflect.go index 5bcfaf66d..47b93662e 100644 --- a/compiler/natives/src/reflect/reflect.go +++ b/compiler/natives/src/reflect/reflect.go @@ -1179,6 +1179,11 @@ func (v Value) Cap() int { return v.typ.Len() case Chan, Slice: return v.object().Get("$capacity").Int() + case Ptr: + if v.typ.Elem().Kind() == Array { + return v.typ.Elem().Len() + } + panic("reflect: call of reflect.Value.Cap on ptr to non-array Value") } panic(&ValueError{"reflect.Value.Cap", k}) } @@ -1405,6 +1410,11 @@ func (v Value) Len() int { return v.object().Get("$buffer").Get("length").Int() case Map: return v.object().Get("size").Int() + case Ptr: + if v.typ.Elem().Kind() == Array { + return v.typ.Elem().Len() + } + panic("reflect: call of reflect.Value.Len on ptr to non-array Value") default: panic(&ValueError{"reflect.Value.Len", k}) } @@ -1450,6 +1460,29 @@ func (v Value) Set(x Value) { v.ptr = x.ptr } +func (v Value) bytesSlow() []byte { + switch v.kind() { + case Slice: + if v.typ.Elem().Kind() != Uint8 { + panic("reflect.Value.Bytes of non-byte slice") + } + return *(*[]byte)(v.ptr) + case Array: + if v.typ.Elem().Kind() != Uint8 { + panic("reflect.Value.Bytes of non-byte array") + } + if !v.CanAddr() { + panic("reflect.Value.Bytes of unaddressable byte array") + } + // Replace the following with JS to avoid using unsafe pointers. + // p := (*byte)(v.ptr) + // n := int((*arrayType)(unsafe.Pointer(v.typ)).len) + // return unsafe.Slice(p, n) + return js.InternalObject(v.ptr).Interface().([]byte) + } + panic(&ValueError{"reflect.Value.Bytes", v.kind()}) +} + func (v Value) SetBytes(x []byte) { v.mustBeAssignable() v.mustBe(Slice) @@ -1728,29 +1761,47 @@ func deepValueEqualJs(v1, v2 Value, visited [][2]unsafe.Pointer) bool { return js.Global.Call("$interfaceIsEqual", js.InternalObject(valueInterface(v1, false)), js.InternalObject(valueInterface(v2, false))).Bool() } -func methodNameSkip() string { - pc, _, _, _ := runtime.Caller(3) - f := runtime.FuncForPC(pc) - if f == nil { - return "unknown method" - } - // Function name extracted from the call stack can be different from vanilla - // Go. Here we try to fix stuff like "Object.$packages.reflect.Q.ptr.SetIterKey" - // into "Value.SetIterKey". - // This workaround may become obsolete after https://github.com/gopherjs/gopherjs/issues/1085 - // is resolved. - name := f.Name() - idx := len(name) - 1 - for idx > 0 { - if name[idx] == '.' { - break +func stringsLastIndex(s string, c byte) int { + for i := len(s) - 1; i >= 0; i-- { + if s[i] == c { + return i } - idx-- } - if idx < 0 { - return name + return -1 +} + +func stringsHasPrefix(s, prefix string) bool { + return len(s) >= len(prefix) && s[:len(prefix)] == prefix +} + +func valueMethodName() string { + var pc [5]uintptr + n := runtime.Callers(1, pc[:]) + frames := runtime.CallersFrames(pc[:n]) + var frame runtime.Frame + for more := true; more; { + frame, more = frames.Next() + name := frame.Function + + // Function name extracted from the call stack can be different from + // vanilla Go, so is not prefixed by "reflect.Value." as needed by the original. + // See https://cs.opensource.google/go/go/+/refs/tags/go1.19.13:src/reflect/value.go;l=173-191 + // Here we try to fix stuff like "Object.$packages.reflect.Q.ptr.SetIterKey" + // into "reflect.Value.SetIterKey". + // This workaround may become obsolete after + // https://github.com/gopherjs/gopherjs/issues/1085 is resolved. + + const prefix = `Object.$packages.reflect.` + if stringsHasPrefix(name, prefix) { + if idx := stringsLastIndex(name, '.'); idx >= 0 { + methodName := name[idx+1:] + if len(methodName) > 0 && 'A' <= methodName[0] && methodName[0] <= 'Z' { + return `reflect.Value.` + methodName + } + } + } } - return "Value" + name[idx:] + return "unknown method" } func verifyNotInHeapPtr(p uintptr) bool { diff --git a/compiler/natives/src/reflect/reflect_test.go b/compiler/natives/src/reflect/reflect_test.go index 112119a4e..79bbe5385 100644 --- a/compiler/natives/src/reflect/reflect_test.go +++ b/compiler/natives/src/reflect/reflect_test.go @@ -285,9 +285,16 @@ func TestMethodCallValueCodePtr(t *testing.T) { t.Skip("methodValueCallCodePtr() is not applicable in GopherJS") } -type B struct{} +//gopherjs:purge for go1.19 without generics +type ( + A struct{} + B[T any] struct{} +) -//gopherjs:prune-original func TestIssue50208(t *testing.T) { t.Skip("This test required generics, which are not yet supported: https://github.com/gopherjs/gopherjs/issues/1013") } + +func TestStructOfTooLarge(t *testing.T) { + t.Skip("This test is dependent on field alignment to determine if a struct size would exceed virtual address space.") +} From 8151b2813eac9a79dc989ea7fa2facb11cd42429 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 30 Jan 2024 17:34:11 -0700 Subject: [PATCH 22/29] Update for fastrand --- compiler/natives/src/net/fastrand.go | 4 ++-- compiler/natives/src/runtime/fastrand.go | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/compiler/natives/src/net/fastrand.go b/compiler/natives/src/net/fastrand.go index 861217a9d..8feafc78f 100644 --- a/compiler/natives/src/net/fastrand.go +++ b/compiler/natives/src/net/fastrand.go @@ -7,5 +7,5 @@ import ( _ "unsafe" // For go:linkname ) -//go:linkname fastrand runtime.fastrand -func fastrand() uint32 +//go:linkname fastrandu runtime.fastrandu +func fastrandu() uint diff --git a/compiler/natives/src/runtime/fastrand.go b/compiler/natives/src/runtime/fastrand.go index 8f6ab6292..a5f2bdbb8 100644 --- a/compiler/natives/src/runtime/fastrand.go +++ b/compiler/natives/src/runtime/fastrand.go @@ -13,3 +13,15 @@ func fastrand() uint32 { // similar distribution. return uint32(js.Global.Get("Math").Call("random").Float() * (1<<32 - 1)) } + +func fastrandn(n uint32) uint32 { + return fastrand() % n +} + +func fastrand64() uint64 { + return uint64(fastrand())<<32 | uint64(fastrand()) +} + +func fastrandu() uint { + return uint(fastrand()) +} From 014360bce172cb58a48916e3981e5896c23953c6 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 30 Jan 2024 17:44:11 -0700 Subject: [PATCH 23/29] Updated net/netip --- compiler/natives/src/net/netip/export_test.go | 2 +- compiler/natives/src/net/netip/fuzz_test.go | 1 + compiler/natives/src/net/netip/netip.go | 3 +-- compiler/natives/src/net/netip/netip_test.go | 10 ++++++++++ 4 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 compiler/natives/src/net/netip/netip_test.go diff --git a/compiler/natives/src/net/netip/export_test.go b/compiler/natives/src/net/netip/export_test.go index 001f348cc..03b7cbe1b 100644 --- a/compiler/natives/src/net/netip/export_test.go +++ b/compiler/natives/src/net/netip/export_test.go @@ -1,4 +1,5 @@ //go:build js +// +build js package netip @@ -8,7 +9,6 @@ import ( "internal/intern" ) -//gopherjs:prune-original func MkAddr(u Uint128, z any) Addr { switch z := z.(type) { case *intern.Value: diff --git a/compiler/natives/src/net/netip/fuzz_test.go b/compiler/natives/src/net/netip/fuzz_test.go index 574201d2d..f7359c5bb 100644 --- a/compiler/natives/src/net/netip/fuzz_test.go +++ b/compiler/natives/src/net/netip/fuzz_test.go @@ -1,4 +1,5 @@ //go:build js +// +build js package netip_test diff --git a/compiler/natives/src/net/netip/netip.go b/compiler/natives/src/net/netip/netip.go index 92a61900a..9d2b8b2d6 100644 --- a/compiler/natives/src/net/netip/netip.go +++ b/compiler/natives/src/net/netip/netip.go @@ -1,4 +1,5 @@ //go:build js +// +build js package netip @@ -17,7 +18,6 @@ var ( z6noz = "\x00ipv6noz" ) -//gopherjs:prune-original func (ip Addr) Zone() string { if ip.z == z4 || ip.z == z6noz { return "" @@ -25,7 +25,6 @@ func (ip Addr) Zone() string { return ip.z } -//gopherjs:prune-original func (ip Addr) WithZone(zone string) Addr { if !ip.Is6() { return ip diff --git a/compiler/natives/src/net/netip/netip_test.go b/compiler/natives/src/net/netip/netip_test.go new file mode 100644 index 000000000..46b116c00 --- /dev/null +++ b/compiler/natives/src/net/netip/netip_test.go @@ -0,0 +1,10 @@ +//go:build js +// +build js + +package netip_test + +import "testing" + +func TestAddrStringAllocs(t *testing.T) { + t.Skip("testing.AllocsPerRun not supported in GopherJS") +} From db5d7b36f2e14c14380942b7515807da61437435 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 30 Jan 2024 17:47:38 -0700 Subject: [PATCH 24/29] Updating sync/atomic --- compiler/natives/src/sync/atomic/atomic_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/compiler/natives/src/sync/atomic/atomic_test.go b/compiler/natives/src/sync/atomic/atomic_test.go index 27ce36df9..dd1bc9994 100644 --- a/compiler/natives/src/sync/atomic/atomic_test.go +++ b/compiler/natives/src/sync/atomic/atomic_test.go @@ -57,9 +57,21 @@ func TestUnaligned64(t *testing.T) { t.Skip("GopherJS emulates atomics, which makes alignment irrelevant.") } +unc TestAutoAligned64(t *testing.T) { + t.Skip("GopherJS emulates atomics, which makes alignment irrelevant.") +} + func TestNilDeref(t *testing.T) { t.Skip("GopherJS does not support generics yet.") } //gopherjs:purge for go1.19 without generics type List struct{} + +func TestHammer32(t *testing.T) { + t.Skip("use of unsafe") +} + +func TestHammer64(t *testing.T) { + t.Skip("use of unsafe") +} From a6b98dd984c684b466e65a8309481762619a07fa Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 30 Jan 2024 17:58:29 -0700 Subject: [PATCH 25/29] Update syscall/js --- compiler/natives/src/syscall/js/export_test.go | 3 +-- compiler/natives/src/syscall/js/js_test.go | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/compiler/natives/src/syscall/js/export_test.go b/compiler/natives/src/syscall/js/export_test.go index 25f6f6833..8f030c4d7 100644 --- a/compiler/natives/src/syscall/js/export_test.go +++ b/compiler/natives/src/syscall/js/export_test.go @@ -4,6 +4,5 @@ package js // Defined to avoid a compile error in the original TestGarbageCollection() -// body. Can't use gopherjs:prune-original on it, since it causes an unused -// import error. +// body. var JSGo Value diff --git a/compiler/natives/src/syscall/js/js_test.go b/compiler/natives/src/syscall/js/js_test.go index c95c2e764..999266da2 100644 --- a/compiler/natives/src/syscall/js/js_test.go +++ b/compiler/natives/src/syscall/js/js_test.go @@ -5,7 +5,6 @@ package js_test import "testing" -//gopherjs:prune-original func TestIntConversion(t *testing.T) { // Same as upstream, but only test cases appropriate for a 32-bit environment. testIntConversion(t, 0) From 967ffcc43e7e9131a908a6848f9c54bba760fcb1 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 30 Jan 2024 18:03:09 -0700 Subject: [PATCH 26/29] Fixed a copy/paste mistake --- compiler/natives/src/sync/atomic/atomic_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/natives/src/sync/atomic/atomic_test.go b/compiler/natives/src/sync/atomic/atomic_test.go index dd1bc9994..e1ec6086c 100644 --- a/compiler/natives/src/sync/atomic/atomic_test.go +++ b/compiler/natives/src/sync/atomic/atomic_test.go @@ -57,7 +57,7 @@ func TestUnaligned64(t *testing.T) { t.Skip("GopherJS emulates atomics, which makes alignment irrelevant.") } -unc TestAutoAligned64(t *testing.T) { +func TestAutoAligned64(t *testing.T) { t.Skip("GopherJS emulates atomics, which makes alignment irrelevant.") } From c08b7bf04917dd4fac1f3265d11ddcd5ce2de087 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 13 Feb 2024 12:12:37 -0700 Subject: [PATCH 27/29] Fixed chocolatey go version --- circle.yml | 2 +- compiler/natives/src/crypto/internal/boring/bbig/big.go | 2 +- compiler/natives/src/crypto/internal/nistec/nistec_test.go | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/circle.yml b/circle.yml index 5c059a6e4..2c3095c8b 100644 --- a/circle.yml +++ b/circle.yml @@ -175,7 +175,7 @@ jobs: - run: name: Install Go command: | - choco install golang --version="<< pipeline.parameters.chocolatey_go_version >>" -my + choco install golang --version="<< pipeline.parameters.chocolatey_go_version >>" -my --force -y go version (Get-Command go).Path [Environment]::SetEnvironmentVariable( diff --git a/compiler/natives/src/crypto/internal/boring/bbig/big.go b/compiler/natives/src/crypto/internal/boring/bbig/big.go index 30ffe1fcd..3a726ba3c 100644 --- a/compiler/natives/src/crypto/internal/boring/bbig/big.go +++ b/compiler/natives/src/crypto/internal/boring/bbig/big.go @@ -33,7 +33,7 @@ func Dec(b boring.BigInt) *big.Int { return new(big.Int) } // Replacing original which uses unsafe: - //x := unsafe.Slice((*big.Word)(&b[0]), len(b)) + // x := unsafe.Slice((*big.Word)(&b[0]), len(b)) x := make([]big.Word, len(b)) for i, w := range b { x[i] = big.Word(w) diff --git a/compiler/natives/src/crypto/internal/nistec/nistec_test.go b/compiler/natives/src/crypto/internal/nistec/nistec_test.go index f5f79398e..d755e7ec3 100644 --- a/compiler/natives/src/crypto/internal/nistec/nistec_test.go +++ b/compiler/natives/src/crypto/internal/nistec/nistec_test.go @@ -4,10 +4,9 @@ package nistec_test import ( - "testing" - "crypto/elliptic" "crypto/internal/nistec" + "testing" ) func TestAllocations(t *testing.T) { From 02aea33b76248f0a35eb7c7044c880fdadf8ee6c Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 13 Feb 2024 13:06:45 -0700 Subject: [PATCH 28/29] Limit augmentation and prune imports to natives that need it --- build/build.go | 42 ++++++++++++++++++++++++++++++++++-------- build/build_test.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/build/build.go b/build/build.go index d3cd32952..def9cd313 100644 --- a/build/build.go +++ b/build/build.go @@ -180,8 +180,12 @@ func parseAndAugment(xctx XContext, pkg *PackageData, isTest bool, fileSet *toke for _, file := range originalFiles { augmentOriginalImports(pkg.ImportPath, file) - augmentOriginalFile(file, overrides) - pruneImports(file) + } + + if len(overrides) > 0 { + for _, file := range originalFiles { + augmentOriginalFile(file, overrides) + } } return append(overlayFiles, originalFiles...), jsFiles, nil @@ -275,6 +279,7 @@ func parserOriginalFiles(pkg *PackageData, fileSet *token.FileSet) ([]*ast.File, // an overlay file AST to collect information such as compiler directives // and perform any initial augmentation needed to the overlay. func augmentOverlayFile(file *ast.File, overrides map[string]overrideInfo) { + anyChange := false for i, decl := range file.Decls { purgeDecl := astutil.Purge(decl) switch d := decl.(type) { @@ -302,15 +307,20 @@ func augmentOverlayFile(file *ast.File, overrides map[string]overrideInfo) { } } if purgeSpec { + anyChange = true d.Specs[j] = nil } } } if purgeDecl { + anyChange = true file.Decls[i] = nil } } - finalizeRemovals(file) + if anyChange { + finalizeRemovals(file) + pruneImports(file) + } } // augmentOriginalImports is the part of parseAndAugment that processes @@ -334,10 +344,12 @@ func augmentOriginalImports(importPath string, file *ast.File) { // original file AST to augment the source code using the overrides from // the overlay files. func augmentOriginalFile(file *ast.File, overrides map[string]overrideInfo) { + anyChange := false for i, decl := range file.Decls { switch d := decl.(type) { case *ast.FuncDecl: if info, ok := overrides[astutil.FuncKey(d)]; ok { + anyChange = true removeFunc := true if info.keepOriginal { // Allow overridden function calls @@ -358,6 +370,7 @@ func augmentOriginalFile(file *ast.File, overrides map[string]overrideInfo) { } else if recvKey := astutil.FuncReceiverKey(d); len(recvKey) > 0 { // check if the receiver has been purged, if so, remove the method too. if info, ok := overrides[recvKey]; ok && info.purgeMethods { + anyChange = true file.Decls[i] = nil } } @@ -366,6 +379,7 @@ func augmentOriginalFile(file *ast.File, overrides map[string]overrideInfo) { switch s := spec.(type) { case *ast.TypeSpec: if _, ok := overrides[s.Name.Name]; ok { + anyChange = true d.Specs[j] = nil } case *ast.ValueSpec: @@ -378,6 +392,7 @@ func augmentOriginalFile(file *ast.File, overrides map[string]overrideInfo) { // to be run, add the call into the overlay. for k, name := range s.Names { if _, ok := overrides[name.Name]; ok { + anyChange = true s.Names[k] = nil s.Values[k] = nil } @@ -405,6 +420,7 @@ func augmentOriginalFile(file *ast.File, overrides map[string]overrideInfo) { } } if removeSpec { + anyChange = true d.Specs[j] = nil } } @@ -413,7 +429,10 @@ func augmentOriginalFile(file *ast.File, overrides map[string]overrideInfo) { } } } - finalizeRemovals(file) + if anyChange { + finalizeRemovals(file) + pruneImports(file) + } } // isOnlyImports determines if this file is empty except for imports. @@ -437,6 +456,14 @@ func isOnlyImports(file *ast.File) bool { // If the removal of code causes an import to be removed, the init's from that // import may not be run anymore. If we still need to run an init for an import // which is no longer used, add it to the overlay as a blank (`_`) import. +// +// This uses the given name or guesses at the name using the import path, +// meaning this doesn't work for packages which have a different package name +// from the path, including those paths which are versioned +// (e.g. `github.com/foo/bar/v2` where the package name is `bar`) +// or if the import is defined using a relative path (e.g. `./..`). +// Those cases don't exist in the native for Go, so we should only run +// this pruning when we have native overlays, but not for unknown packages. func pruneImports(file *ast.File) { if isOnlyImports(file) && !astutil.HasDirectivePrefix(file, `//go:linkname `) { // The file is empty, remove all imports including any `.` or `_` imports. @@ -478,12 +505,11 @@ func pruneImports(file *ast.File) { // since the import is otherwise unused set the name to blank. in.Name = ast.NewIdent(`_`) delete(unused, name) - if len(unused) == 0 { - return - } - break } } + if len(unused) == 0 { + return + } // Remove all unused import specifications isUnusedSpec := map[*ast.ImportSpec]bool{} diff --git a/build/build_test.go b/build/build_test.go index 5f025540b..343e8b933 100644 --- a/build/build_test.go +++ b/build/build_test.go @@ -680,6 +680,39 @@ func TestOriginalAugmentation(t *testing.T) { import _ "unsafe"`, want: `//go:linkname foo bar import _ "unsafe"`, + }, { + desc: `multiple imports for directives`, + info: map[string]overrideInfo{ + `A`: {}, + `C`: {}, + }, + src: `import "unsafe" + import "embed" + + //go:embed hello.txt + var A embed.FS + + //go:embed goodbye.txt + var B string + + var C unsafe.Pointer + + // override Now with hardcoded time for testing + //go:linkname timeNow time.Now + func timeNow() time.Time { + return time.Date(2012, 8, 6, 0, 0, 0, 0, time.UTC) + }`, + want: `import _ "unsafe" + import _ "embed" + + //go:embed goodbye.txt + var B string + + // override Now with hardcoded time for testing + //go:linkname timeNow time.Now + func timeNow() time.Time { + return time.Date(2012, 8, 6, 0, 0, 0, 0, time.UTC) + }`, }, } From 1011858069d07c9dae6799f189033af48d9acd31 Mon Sep 17 00:00:00 2001 From: Nevkontakte Date: Sat, 24 Feb 2024 13:45:51 +0000 Subject: [PATCH 29/29] Update GopherJS version to 1.19.0-beta1 in perparation for release. --- README.md | 2 +- compiler/version_check.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f29bb9084..3804dbf0f 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ version, you can use an [older GopherJS release](https://github.com/gopherjs/gop Install GopherJS with `go install`: ``` -go install github.com/gopherjs/gopherjs@v1.19.0-alpha1 # Or replace 'v1.19.0-alpha1' with another version. +go install github.com/gopherjs/gopherjs@v1.19.0-beta1 # Or replace 'v1.19.0-beta1' with another version. ``` If your local Go distribution as reported by `go version` is newer than Go 1.19, then you need to set the `GOPHERJS_GOROOT` environment variable to a directory that contains a Go 1.19 distribution. For example: diff --git a/compiler/version_check.go b/compiler/version_check.go index 36bd4acd3..d672fa45a 100644 --- a/compiler/version_check.go +++ b/compiler/version_check.go @@ -12,7 +12,7 @@ import ( ) // Version is the GopherJS compiler version string. -const Version = "1.19.0-alpha1+go1.19.13" +const Version = "1.19.0-beta1+go1.19.13" // GoVersion is the current Go 1.x version that GopherJS is compatible with. const GoVersion = 19