From 7681df9add1a37164892dafc5f0d00ff31eb2343 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 5 Dec 2023 15:59:55 -0700 Subject: [PATCH 01/11] Adding directive ast util --- compiler/astutil/astutil.go | 74 +++++++--- compiler/astutil/astutil_test.go | 233 ++++++++++++++++++++++++++++++ internal/srctesting/srctesting.go | 42 +++++- 3 files changed, 323 insertions(+), 26 deletions(-) diff --git a/compiler/astutil/astutil.go b/compiler/astutil/astutil.go index 30febe1cb..9089ebdd3 100644 --- a/compiler/astutil/astutil.go +++ b/compiler/astutil/astutil.go @@ -5,7 +5,7 @@ import ( "go/ast" "go/token" "go/types" - "strings" + "regexp" ) func RemoveParens(e ast.Expr) ast.Expr { @@ -72,6 +72,58 @@ func FuncKey(d *ast.FuncDecl) string { return recv.(*ast.Ident).Name + "." + d.Name.Name } +// anyDocLine calls the given predicate on all associated documentation +// lines and line-comment lines from the given node. +// If the predicate returns true for any line then true is returned. +func anyDocLine(node any, predicate func(line string) bool) bool { + switch a := node.(type) { + case *ast.Comment: + return a != nil && predicate(a.Text) + case *ast.CommentGroup: + if a != nil { + for _, c := range a.List { + if anyDocLine(c, predicate) { + return true + } + } + } + return false + case *ast.Field: + return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) + case *ast.File: + return a != nil && anyDocLine(a.Doc, predicate) + case *ast.FuncDecl: + return a != nil && anyDocLine(a.Doc, predicate) + case *ast.GenDecl: + return a != nil && anyDocLine(a.Doc, predicate) + case *ast.ImportSpec: + return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) + case *ast.TypeSpec: + return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) + case *ast.ValueSpec: + return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) + default: + panic(fmt.Errorf(`unexpected node type to get doc from: %v`, node)) + } +} + +// directiveMatcher is a regex which matches a GopherJS directive +// and finds the directive action. +// +// This matches the largest directive action until whitespace or EOL +// to differentiate from any directive action which is a prefix +// for another directive action. +var directiveMatcher = regexp.MustCompile(`^//gopherjs:(\S+)`) + +// hasDirective returns true if the associated documentation +// or line comments for the given node have the given directive action. +func hasDirective(node any, directive string) bool { + return anyDocLine(node, func(line string) bool { + m := directiveMatcher.FindStringSubmatch(line) + return len(m) >= 2 && m[1] == directive + }) +} + // PruneOriginal returns true if gopherjs:prune-original directive is present // before a function decl. // @@ -82,15 +134,7 @@ func FuncKey(d *ast.FuncDecl) string { // such as code expecting ints to be 64-bit. It should be used with caution // since it may create unused imports in the original source file. func PruneOriginal(d *ast.FuncDecl) bool { - if d.Doc == nil { - return false - } - for _, c := range d.Doc.List { - if strings.HasPrefix(c.Text, "//gopherjs:prune-original") { - return true - } - } - return false + return hasDirective(d, `prune-original`) } // KeepOriginal returns true if gopherjs:keep-original directive is present @@ -102,15 +146,7 @@ func PruneOriginal(d *ast.FuncDecl) bool { // function in the original called `foo`, it will be accessible by the name // `_gopherjs_original_foo`. func KeepOriginal(d *ast.FuncDecl) bool { - if d.Doc == nil { - return false - } - for _, c := range d.Doc.List { - if strings.HasPrefix(c.Text, "//gopherjs:keep-original") { - return true - } - } - return false + return hasDirective(d, `keep-original`) } // FindLoopStmt tries to find the loop statement among the AST nodes in the diff --git a/compiler/astutil/astutil_test.go b/compiler/astutil/astutil_test.go index a996ae73f..84ad5d18c 100644 --- a/compiler/astutil/astutil_test.go +++ b/compiler/astutil/astutil_test.go @@ -130,6 +130,239 @@ func TestPruneOriginal(t *testing.T) { } } +func TestHasDirectiveOnDecl(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: `no comment on function`, + src: `package testpackage; + func foo() {}`, + want: false, + }, { + desc: `no directive on function with comment`, + src: `package testpackage; + // foo has no directive + func foo() {}`, + want: false, + }, { + desc: `wrong directive on function`, + src: `package testpackage; + //gopherjs:wrong-directive + func foo() {}`, + want: false, + }, { + desc: `correct directive on function`, + src: `package testpackage; + //gopherjs:do-stuff + func foo() {}`, + want: true, + }, { + desc: `prefix directive on function`, + src: `package testpackage; + //gopherjs:do-stuffs + func foo() {}`, + want: false, + }, { + desc: `multiple directives on function`, + src: `package testpackage; + //gopherjs:wrong-directive + //gopherjs:do-stuff + //gopherjs:another-directive + func foo() {}`, + want: true, + }, { + desc: `directive with explanation on function`, + src: `package testpackage; + //gopherjs:do-stuff 'cause we can + func foo() {}`, + want: true, + }, { + desc: `no directive on type declaration`, + src: `package testpackage; + // Foo has a comment + type Foo int`, + want: false, + }, { + desc: `directive on type declaration`, + src: `package testpackage; + //gopherjs:do-stuff + type Foo int`, + want: true, + }, { + desc: `no directive on const declaration`, + src: `package testpackage; + const foo = 42`, + want: false, + }, { + desc: `directive on const documentation`, + src: `package testpackage; + //gopherjs:do-stuff + const foo = 42`, + want: true, + }, { + desc: `no directive on var declaration`, + src: `package testpackage; + var foo = 42`, + want: false, + }, { + desc: `directive on var documentation`, + src: `package testpackage; + //gopherjs:do-stuff + var foo = 42`, + want: true, + }, { + desc: `no directive on var declaration`, + src: `package testpackage; + import _ "embed"`, + want: false, + }, { + desc: `directive on var documentation`, + src: `package testpackage; + //gopherjs:do-stuff + import _ "embed"`, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + decl := srctesting.ParseDecl(t, test.src) + if got := hasDirective(decl, action); got != test.want { + t.Errorf("hasDirective(%T, %q) returned %t, want %t", decl, action, got, test.want) + } + }) + } +} + +func TestHasDirectiveOnSpec(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: `no directive on type specification`, + src: `package testpackage; + type Foo int`, + want: false, + }, { + desc: `directive in doc on type specification`, + src: `package testpackage; + type ( + //gopherjs:do-stuff + Foo int + )`, + want: true, + }, { + desc: `directive in line on type specification`, + src: `package testpackage; + type Foo int //gopherjs:do-stuff`, + want: true, + }, { + desc: `no directive on const specification`, + src: `package testpackage; + const foo = 42`, + want: false, + }, { + desc: `directive in doc on const specification`, + src: `package testpackage; + const ( + //gopherjs:do-stuff + foo = 42 + )`, + want: true, + }, { + desc: `directive in line on const specification`, + src: `package testpackage; + const foo = 42 //gopherjs:do-stuff`, + want: true, + }, { + desc: `no directive on var specification`, + src: `package testpackage; + var foo = 42`, + want: false, + }, { + desc: `directive in doc on var specification`, + src: `package testpackage; + var ( + //gopherjs:do-stuff + foo = 42 + )`, + want: true, + }, { + desc: `directive in line on var specification`, + src: `package testpackage; + var foo = 42 //gopherjs:do-stuff`, + want: true, + }, { + desc: `no directive on import specification`, + src: `package testpackage; + import _ "embed"`, + want: false, + }, { + desc: `directive in doc on import specification`, + src: `package testpackage; + import ( + //gopherjs:do-stuff + _ "embed" + )`, + want: true, + }, { + desc: `directive in line on import specification`, + src: `package testpackage; + import _ "embed" //gopherjs:do-stuff`, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + spec := srctesting.ParseSpec(t, test.src) + if got := hasDirective(spec, action); got != test.want { + t.Errorf("hasDirective(%T, %q) returned %t, want %t", spec, action, got, test.want) + } + }) + } +} + +func TestHasDirectiveOnFile(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: `no directive on file`, + src: `package testpackage; + //gopherjs:do-stuff + type Foo int`, + want: false, + }, { + desc: `directive on file`, + src: `//gopherjs:do-stuff + package testpackage; + type Foo int`, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + fset := token.NewFileSet() + file := srctesting.Parse(t, fset, test.src) + if got := hasDirective(file, action); got != test.want { + t.Errorf("hasDirective(%T, %q) returned %t, want %t", file, action, got, test.want) + } + }) + } +} + func TestEndsWithReturn(t *testing.T) { tests := []struct { desc string diff --git a/internal/srctesting/srctesting.go b/internal/srctesting/srctesting.go index 1d9cecd20..c9ff63575 100644 --- a/internal/srctesting/srctesting.go +++ b/internal/srctesting/srctesting.go @@ -48,24 +48,52 @@ func Check(t *testing.T, fset *token.FileSet, files ...*ast.File) (*types.Info, return typesInfo, typesPkg } -// ParseFuncDecl parses source with a single function defined and returns the -// function AST. +// ParseDecl parses source with a single declaration and +// returns that declaration AST. // -// Fails the test if there isn't exactly one function declared in the source. -func ParseFuncDecl(t *testing.T, src string) *ast.FuncDecl { +// Fails the test if there isn't exactly one declaration in the source. +func ParseDecl(t *testing.T, src string) ast.Decl { t.Helper() fset := token.NewFileSet() file := Parse(t, fset, src) if l := len(file.Decls); l != 1 { - t.Fatalf("Got %d decls in the sources, expected exactly 1", l) + t.Fatalf(`Got %d decls in the sources, expected exactly 1`, l) } - fdecl, ok := file.Decls[0].(*ast.FuncDecl) + return file.Decls[0] +} + +// ParseFuncDecl parses source with a single function defined and returns the +// function AST. +// +// Fails the test if there isn't exactly one function declared in the source. +func ParseFuncDecl(t *testing.T, src string) *ast.FuncDecl { + t.Helper() + decl := ParseDecl(t, src) + fdecl, ok := decl.(*ast.FuncDecl) if !ok { - t.Fatalf("Got %T decl, expected *ast.FuncDecl", file.Decls[0]) + t.Fatalf("Got %T decl, expected *ast.FuncDecl", decl) } return fdecl } +// ParseSpec parses source with a single declaration containing +// a single specification and returns that specification AST. +// +// Fails the test if there isn't exactly one declaration and +// one specification in the source. +func ParseSpec(t *testing.T, src string) ast.Spec { + t.Helper() + decl := ParseDecl(t, src) + gdecl, ok := decl.(*ast.GenDecl) + if !ok { + t.Fatalf("Got %T decl, expected *ast.GenDecl", decl) + } + if l := len(gdecl.Specs); l != 1 { + t.Fatalf(`Got %d spec in the sources, expected exactly 1`, l) + } + return gdecl.Specs[0] +} + // Format AST node into a string. // // The node type must be *ast.File, *printer.CommentedNode, []ast.Decl, From d486c248c886321eb888d35042456c687c082c40 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Wed, 6 Dec 2023 13:57:15 -0700 Subject: [PATCH 02/11] Added directives --- compiler/astutil/astutil.go | 78 ------ compiler/astutil/astutil_test.go | 281 ------------------- compiler/astutil/directives.go | 104 +++++++ compiler/astutil/directives_test.go | 420 ++++++++++++++++++++++++++++ 4 files changed, 524 insertions(+), 359 deletions(-) create mode 100644 compiler/astutil/directives.go create mode 100644 compiler/astutil/directives_test.go diff --git a/compiler/astutil/astutil.go b/compiler/astutil/astutil.go index 9089ebdd3..84482f4cc 100644 --- a/compiler/astutil/astutil.go +++ b/compiler/astutil/astutil.go @@ -5,7 +5,6 @@ import ( "go/ast" "go/token" "go/types" - "regexp" ) func RemoveParens(e ast.Expr) ast.Expr { @@ -72,83 +71,6 @@ func FuncKey(d *ast.FuncDecl) string { return recv.(*ast.Ident).Name + "." + d.Name.Name } -// anyDocLine calls the given predicate on all associated documentation -// lines and line-comment lines from the given node. -// If the predicate returns true for any line then true is returned. -func anyDocLine(node any, predicate func(line string) bool) bool { - switch a := node.(type) { - case *ast.Comment: - return a != nil && predicate(a.Text) - case *ast.CommentGroup: - if a != nil { - for _, c := range a.List { - if anyDocLine(c, predicate) { - return true - } - } - } - return false - case *ast.Field: - return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) - case *ast.File: - return a != nil && anyDocLine(a.Doc, predicate) - case *ast.FuncDecl: - return a != nil && anyDocLine(a.Doc, predicate) - case *ast.GenDecl: - return a != nil && anyDocLine(a.Doc, predicate) - case *ast.ImportSpec: - return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) - case *ast.TypeSpec: - return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) - case *ast.ValueSpec: - return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) - default: - panic(fmt.Errorf(`unexpected node type to get doc from: %v`, node)) - } -} - -// directiveMatcher is a regex which matches a GopherJS directive -// and finds the directive action. -// -// This matches the largest directive action until whitespace or EOL -// to differentiate from any directive action which is a prefix -// for another directive action. -var directiveMatcher = regexp.MustCompile(`^//gopherjs:(\S+)`) - -// hasDirective returns true if the associated documentation -// or line comments for the given node have the given directive action. -func hasDirective(node any, directive string) bool { - return anyDocLine(node, func(line string) bool { - m := directiveMatcher.FindStringSubmatch(line) - return len(m) >= 2 && m[1] == directive - }) -} - -// PruneOriginal returns true if gopherjs:prune-original directive is present -// before a function decl. -// -// `//gopherjs:prune-original` is a GopherJS-specific directive, which can be -// applied to functions in native overlays and will instruct the augmentation -// logic to delete the body of a standard library function that was replaced. -// This directive can be used to remove code that would be invalid in GopherJS, -// such as code expecting ints to be 64-bit. It should be used with caution -// since it may create unused imports in the original source file. -func PruneOriginal(d *ast.FuncDecl) bool { - return hasDirective(d, `prune-original`) -} - -// KeepOriginal returns true if gopherjs:keep-original directive is present -// before a function decl. -// -// `//gopherjs:keep-original` is a GopherJS-specific directive, which can be -// applied to functions in native overlays and will instruct the augmentation -// logic to expose the original function such that it can be called. For a -// function in the original called `foo`, it will be accessible by the name -// `_gopherjs_original_foo`. -func KeepOriginal(d *ast.FuncDecl) bool { - return hasDirective(d, `keep-original`) -} - // FindLoopStmt tries to find the loop statement among the AST nodes in the // |stack| that corresponds to the break/continue statement represented by // branch. diff --git a/compiler/astutil/astutil_test.go b/compiler/astutil/astutil_test.go index 84ad5d18c..f13ae0a1b 100644 --- a/compiler/astutil/astutil_test.go +++ b/compiler/astutil/astutil_test.go @@ -82,287 +82,6 @@ func TestFuncKey(t *testing.T) { } } -func TestPruneOriginal(t *testing.T) { - tests := []struct { - desc string - src string - want bool - }{ - { - desc: "no comment", - src: `package testpackage; - func foo() {}`, - want: false, - }, { - desc: "regular godoc", - src: `package testpackage; - // foo does something - func foo() {}`, - want: false, - }, { - desc: "only directive", - src: `package testpackage; - //gopherjs:prune-original - func foo() {}`, - want: true, - }, { - desc: "directive with explanation", - src: `package testpackage; - //gopherjs:prune-original because reasons - func foo() {}`, - want: true, - }, { - desc: "directive in godoc", - src: `package testpackage; - // foo does something - //gopherjs:prune-original - func foo() {}`, - want: true, - }, - } - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - fdecl := srctesting.ParseFuncDecl(t, test.src) - if got := PruneOriginal(fdecl); got != test.want { - t.Errorf("PruneOriginal() returned %t, want %t", got, test.want) - } - }) - } -} - -func TestHasDirectiveOnDecl(t *testing.T) { - tests := []struct { - desc string - src string - want bool - }{ - { - desc: `no comment on function`, - src: `package testpackage; - func foo() {}`, - want: false, - }, { - desc: `no directive on function with comment`, - src: `package testpackage; - // foo has no directive - func foo() {}`, - want: false, - }, { - desc: `wrong directive on function`, - src: `package testpackage; - //gopherjs:wrong-directive - func foo() {}`, - want: false, - }, { - desc: `correct directive on function`, - src: `package testpackage; - //gopherjs:do-stuff - func foo() {}`, - want: true, - }, { - desc: `prefix directive on function`, - src: `package testpackage; - //gopherjs:do-stuffs - func foo() {}`, - want: false, - }, { - desc: `multiple directives on function`, - src: `package testpackage; - //gopherjs:wrong-directive - //gopherjs:do-stuff - //gopherjs:another-directive - func foo() {}`, - want: true, - }, { - desc: `directive with explanation on function`, - src: `package testpackage; - //gopherjs:do-stuff 'cause we can - func foo() {}`, - want: true, - }, { - desc: `no directive on type declaration`, - src: `package testpackage; - // Foo has a comment - type Foo int`, - want: false, - }, { - desc: `directive on type declaration`, - src: `package testpackage; - //gopherjs:do-stuff - type Foo int`, - want: true, - }, { - desc: `no directive on const declaration`, - src: `package testpackage; - const foo = 42`, - want: false, - }, { - desc: `directive on const documentation`, - src: `package testpackage; - //gopherjs:do-stuff - const foo = 42`, - want: true, - }, { - desc: `no directive on var declaration`, - src: `package testpackage; - var foo = 42`, - want: false, - }, { - desc: `directive on var documentation`, - src: `package testpackage; - //gopherjs:do-stuff - var foo = 42`, - want: true, - }, { - desc: `no directive on var declaration`, - src: `package testpackage; - import _ "embed"`, - want: false, - }, { - desc: `directive on var documentation`, - src: `package testpackage; - //gopherjs:do-stuff - import _ "embed"`, - want: true, - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - const action = `do-stuff` - decl := srctesting.ParseDecl(t, test.src) - if got := hasDirective(decl, action); got != test.want { - t.Errorf("hasDirective(%T, %q) returned %t, want %t", decl, action, got, test.want) - } - }) - } -} - -func TestHasDirectiveOnSpec(t *testing.T) { - tests := []struct { - desc string - src string - want bool - }{ - { - desc: `no directive on type specification`, - src: `package testpackage; - type Foo int`, - want: false, - }, { - desc: `directive in doc on type specification`, - src: `package testpackage; - type ( - //gopherjs:do-stuff - Foo int - )`, - want: true, - }, { - desc: `directive in line on type specification`, - src: `package testpackage; - type Foo int //gopherjs:do-stuff`, - want: true, - }, { - desc: `no directive on const specification`, - src: `package testpackage; - const foo = 42`, - want: false, - }, { - desc: `directive in doc on const specification`, - src: `package testpackage; - const ( - //gopherjs:do-stuff - foo = 42 - )`, - want: true, - }, { - desc: `directive in line on const specification`, - src: `package testpackage; - const foo = 42 //gopherjs:do-stuff`, - want: true, - }, { - desc: `no directive on var specification`, - src: `package testpackage; - var foo = 42`, - want: false, - }, { - desc: `directive in doc on var specification`, - src: `package testpackage; - var ( - //gopherjs:do-stuff - foo = 42 - )`, - want: true, - }, { - desc: `directive in line on var specification`, - src: `package testpackage; - var foo = 42 //gopherjs:do-stuff`, - want: true, - }, { - desc: `no directive on import specification`, - src: `package testpackage; - import _ "embed"`, - want: false, - }, { - desc: `directive in doc on import specification`, - src: `package testpackage; - import ( - //gopherjs:do-stuff - _ "embed" - )`, - want: true, - }, { - desc: `directive in line on import specification`, - src: `package testpackage; - import _ "embed" //gopherjs:do-stuff`, - want: true, - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - const action = `do-stuff` - spec := srctesting.ParseSpec(t, test.src) - if got := hasDirective(spec, action); got != test.want { - t.Errorf("hasDirective(%T, %q) returned %t, want %t", spec, action, got, test.want) - } - }) - } -} - -func TestHasDirectiveOnFile(t *testing.T) { - tests := []struct { - desc string - src string - want bool - }{ - { - desc: `no directive on file`, - src: `package testpackage; - //gopherjs:do-stuff - type Foo int`, - want: false, - }, { - desc: `directive on file`, - src: `//gopherjs:do-stuff - package testpackage; - type Foo int`, - want: true, - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - const action = `do-stuff` - fset := token.NewFileSet() - file := srctesting.Parse(t, fset, test.src) - if got := hasDirective(file, action); got != test.want { - t.Errorf("hasDirective(%T, %q) returned %t, want %t", file, action, got, test.want) - } - }) - } -} - func TestEndsWithReturn(t *testing.T) { tests := []struct { desc string diff --git a/compiler/astutil/directives.go b/compiler/astutil/directives.go new file mode 100644 index 000000000..088b0371c --- /dev/null +++ b/compiler/astutil/directives.go @@ -0,0 +1,104 @@ +package astutil + +import ( + "fmt" + "go/ast" + "regexp" +) + +// PruneOriginal returns true if gopherjs:prune-original directive is present +// before a function decl. +// +// `//gopherjs:prune-original` is a GopherJS-specific directive, which can be +// applied to functions in native overlays and will instruct the augmentation +// logic to delete the body of a standard library function that was replaced. +// This directive can be used to remove code that would be invalid in GopherJS, +// such as code expecting ints to be 64-bit. It should be used with caution +// since it may create unused imports in the original source file. +func PruneOriginal(d *ast.FuncDecl) bool { + return hasDirective(d, `prune-original`) +} + +// KeepOriginal returns true if gopherjs:keep-original directive is present +// before a function decl. +// +// `//gopherjs:keep-original` is a GopherJS-specific directive, which can be +// applied to functions in native overlays and will instruct the augmentation +// logic to expose the original function such that it can be called. For a +// function in the original called `foo`, it will be accessible by the name +// `_gopherjs_original_foo`. +func KeepOriginal(d *ast.FuncDecl) bool { + return hasDirective(d, `keep-original`) +} + +// PurgeOriginal returns true if gopherjs:purge-original directive is present +// on a struct, interface, variables, constants, and functions. +// +// `//gopherjs:purge-original` is a GopherJS-specific directive, which can be +// applied in native overlays and will instruct the augmentation logic to +// delete part of the standard library without a replacement. This directive +// can be used to remove code that would be invalid in GopherJS, such as code +// using unsupported features (e.g. generic interfaces and methods before +// generics were fully supported). +// It should be used with caution since it may remove needed dependencies. +func PurgeOriginal(d any) bool { + return hasDirective(d, `purge-original`) +} + +// anyDocLine calls the given predicate on all associated documentation +// lines and line-comment lines from the given node. +// If the predicate returns true for any line then true is returned. +func anyDocLine(node any, predicate func(line string) bool) bool { + switch a := node.(type) { + case *ast.Comment: + return a != nil && predicate(a.Text) + case *ast.CommentGroup: + if a != nil { + for _, c := range a.List { + if anyDocLine(c, predicate) { + return true + } + } + } + return false + case *ast.Field: + return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) + case *ast.File: + return a != nil && anyDocLine(a.Doc, predicate) + case *ast.FuncDecl: + return a != nil && anyDocLine(a.Doc, predicate) + case *ast.GenDecl: + return a != nil && anyDocLine(a.Doc, predicate) + case *ast.ImportSpec: + return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) + case *ast.TypeSpec: + return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) + case *ast.ValueSpec: + return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) + default: + panic(fmt.Errorf(`unexpected node type to get doc from: %T`, node)) + } +} + +// directiveMatcher is a regex which matches a GopherJS directive +// and finds the directive action. +// +// This matches the largest directive action until whitespace or EOL +// to differentiate from any directive action which is a prefix +// for another directive action. +var directiveMatcher = regexp.MustCompile(`^\/(?:\/|\*)gopherjs:([\w-]+)`) + +// hasDirective returns true if the associated documentation +// or line comments for the given node have the given directive action. +// +// All gopherjs directives must start with `//gopherjs:` or `/*gopherjs:` +// and followed by an action without any whitespace. The action must be +// one or more letter, decimal, underscore, or hyphen. +// +// see https://pkg.go.dev/cmd/compile#hdr-Compiler_Directives +func hasDirective(node any, directive string) bool { + return anyDocLine(node, func(line string) bool { + m := directiveMatcher.FindStringSubmatch(line) + return len(m) == 2 && m[1] == directive + }) +} diff --git a/compiler/astutil/directives_test.go b/compiler/astutil/directives_test.go new file mode 100644 index 000000000..9325f73f4 --- /dev/null +++ b/compiler/astutil/directives_test.go @@ -0,0 +1,420 @@ +package astutil + +import ( + "fmt" + "go/ast" + "go/token" + "testing" + + "github.com/gopherjs/gopherjs/internal/srctesting" +) + +func TestPruneOriginal(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: "no comment", + src: `package testpackage; + func foo() {}`, + want: false, + }, { + desc: "regular godoc", + src: `package testpackage; + // foo does something + func foo() {}`, + want: false, + }, { + desc: "only directive", + src: `package testpackage; + //gopherjs:prune-original + func foo() {}`, + want: true, + }, { + desc: "directive with explanation", + src: `package testpackage; + //gopherjs:prune-original because reasons + func foo() {}`, + want: true, + }, { + desc: "directive in godoc", + src: `package testpackage; + // foo does something + //gopherjs:prune-original + func foo() {}`, + want: true, + }, + } + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + fdecl := srctesting.ParseFuncDecl(t, test.src) + if got := PruneOriginal(fdecl); got != test.want { + t.Errorf("PruneOriginal() returned %t, want %t", got, test.want) + } + }) + } +} + +func TestHasDirectiveOnDecl(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: `no comment on function`, + src: `package testpackage; + func foo() {}`, + want: false, + }, { + desc: `no directive on function with comment`, + src: `package testpackage; + // foo has no directive + func foo() {}`, + want: false, + }, { + desc: `wrong directive on function`, + src: `package testpackage; + //gopherjs:wrong-directive + func foo() {}`, + want: false, + }, { + desc: `correct directive on function`, + src: `package testpackage; + //gopherjs:do-stuff + // foo has a directive to do stuff + func foo() {}`, + want: true, + }, { + desc: `correct directive in multiline comment on function`, + src: `package testpackage; + /*gopherjs:do-stuff + foo has a directive to do stuff + */ + func foo() {}`, + want: true, + }, { + desc: `invalid directive in multiline comment on function`, + src: `package testpackage; + /* + gopherjs:do-stuff + */ + func foo() {}`, + want: false, + }, { + desc: `prefix directive on function`, + src: `package testpackage; + //gopherjs:do-stuffs + func foo() {}`, + want: false, + }, { + desc: `multiple directives on function`, + src: `package testpackage; + //gopherjs:wrong-directive + //gopherjs:do-stuff + //gopherjs:another-directive + func foo() {}`, + want: true, + }, { + desc: `directive with explanation on function`, + src: `package testpackage; + //gopherjs:do-stuff 'cause we can + func foo() {}`, + want: true, + }, { + desc: `no directive on type declaration`, + src: `package testpackage; + // Foo has a comment + type Foo int`, + want: false, + }, { + desc: `directive on type declaration`, + src: `package testpackage; + //gopherjs:do-stuff + type Foo int`, + want: true, + }, { + desc: `no directive on const declaration`, + src: `package testpackage; + const foo = 42`, + want: false, + }, { + desc: `directive on const documentation`, + src: `package testpackage; + //gopherjs:do-stuff + const foo = 42`, + want: true, + }, { + desc: `no directive on var declaration`, + src: `package testpackage; + var foo = 42`, + want: false, + }, { + desc: `directive on var documentation`, + src: `package testpackage; + //gopherjs:do-stuff + var foo = 42`, + want: true, + }, { + desc: `no directive on var declaration`, + src: `package testpackage; + import _ "embed"`, + want: false, + }, { + desc: `directive on var documentation`, + src: `package testpackage; + //gopherjs:do-stuff + import _ "embed"`, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + decl := srctesting.ParseDecl(t, test.src) + if got := hasDirective(decl, action); got != test.want { + t.Errorf(`hasDirective(%T, %q) returned %t, want %t`, decl, action, got, test.want) + } + }) + } +} + +func TestHasDirectiveOnSpec(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: `no directive on type specification`, + src: `package testpackage; + type Foo int`, + want: false, + }, { + desc: `directive in doc on type specification`, + src: `package testpackage; + type ( + //gopherjs:do-stuff + Foo int + )`, + want: true, + }, { + desc: `directive in line on type specification`, + src: `package testpackage; + type Foo int //gopherjs:do-stuff`, + want: true, + }, { + desc: `no directive on const specification`, + src: `package testpackage; + const foo = 42`, + want: false, + }, { + desc: `directive in doc on const specification`, + src: `package testpackage; + const ( + //gopherjs:do-stuff + foo = 42 + )`, + want: true, + }, { + desc: `directive in line on const specification`, + src: `package testpackage; + const foo = 42 //gopherjs:do-stuff`, + want: true, + }, { + desc: `no directive on var specification`, + src: `package testpackage; + var foo = 42`, + want: false, + }, { + desc: `directive in doc on var specification`, + src: `package testpackage; + var ( + //gopherjs:do-stuff + foo = 42 + )`, + want: true, + }, { + desc: `directive in line on var specification`, + src: `package testpackage; + var foo = 42 //gopherjs:do-stuff`, + want: true, + }, { + desc: `no directive on import specification`, + src: `package testpackage; + import _ "embed"`, + want: false, + }, { + desc: `directive in doc on import specification`, + src: `package testpackage; + import ( + //gopherjs:do-stuff + _ "embed" + )`, + want: true, + }, { + desc: `directive in line on import specification`, + src: `package testpackage; + import _ "embed" //gopherjs:do-stuff`, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + spec := srctesting.ParseSpec(t, test.src) + if got := hasDirective(spec, action); got != test.want { + t.Errorf(`hasDirective(%T, %q) returned %t, want %t`, spec, action, got, test.want) + } + }) + } +} + +func TestHasDirectiveOnFile(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: `no directive on file`, + src: `package testpackage; + //gopherjs:do-stuff + type Foo int`, + want: false, + }, { + desc: `directive on file`, + src: `//gopherjs:do-stuff + package testpackage; + type Foo int`, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + fset := token.NewFileSet() + file := srctesting.Parse(t, fset, test.src) + if got := hasDirective(file, action); got != test.want { + t.Errorf(`hasDirective(%T, %q) returned %t, want %t`, file, action, got, test.want) + } + }) + } +} + +func TestHasDirectiveOnField(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: `no directive on struct field`, + src: `package testpackage; + type Foo struct { + bar int + }`, + want: false, + }, { + desc: `directive in doc on struct field`, + src: `package testpackage; + type Foo struct { + //gopherjs:do-stuff + bar int + }`, + want: true, + }, { + desc: `directive in line on struct field`, + src: `package testpackage; + type Foo struct { + bar int //gopherjs:do-stuff + }`, + want: true, + }, { + desc: `no directive on interface method`, + src: `package testpackage; + type Foo interface { + Bar(a int) int + }`, + want: false, + }, { + desc: `directive in doc on interface method`, + src: `package testpackage; + type Foo interface { + //gopherjs:do-stuff + Bar(a int) int + }`, + want: true, + }, { + desc: `directive in line on interface method`, + src: `package testpackage; + type Foo interface { + Bar(a int) int //gopherjs:do-stuff + }`, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + spec := srctesting.ParseSpec(t, test.src) + tspec := spec.(*ast.TypeSpec) + var field *ast.Field + switch typeNode := tspec.Type.(type) { + case *ast.StructType: + field = typeNode.Fields.List[0] + case *ast.InterfaceType: + field = typeNode.Methods.List[0] + default: + t.Errorf(`unexpected node type, %T, when finding field`, typeNode) + return + } + if got := hasDirective(field, action); got != test.want { + t.Errorf(`hasDirective(%T, %q) returned %t, want %t`, field, action, got, test.want) + } + }) + } +} + +func TestHasDirectiveBadCase(t *testing.T) { + tests := []struct { + desc string + node any + want string + }{ + { + desc: `untyped nil node`, + node: nil, + want: `unexpected node type to get doc from: `, + }, { + desc: `unexpected node type`, + node: &ast.ArrayType{}, + want: `unexpected node type to get doc from: *ast.ArrayType`, + }, { + desc: `nil expected node type`, + node: (*ast.FuncDecl)(nil), + want: ``, // no panic + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + + var got string + func() { + defer func() { got = fmt.Sprint(recover()) }() + hasDirective(test.node, action) + }() + if got != test.want { + t.Errorf(`hasDirective(%T, %q) returned %s, want %s`, test.node, action, got, test.want) + } + }) + } +} From 892b5e228c80969e82c97e4014cd14b27af5734c Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Wed, 6 Dec 2023 14:39:17 -0700 Subject: [PATCH 03/11] Added directives --- compiler/astutil/astutil.go | 98 +++++++ compiler/astutil/astutil_test.go | 412 +++++++++++++++++++++++++++ compiler/astutil/directives.go | 104 ------- compiler/astutil/directives_test.go | 420 ---------------------------- 4 files changed, 510 insertions(+), 524 deletions(-) delete mode 100644 compiler/astutil/directives.go delete mode 100644 compiler/astutil/directives_test.go diff --git a/compiler/astutil/astutil.go b/compiler/astutil/astutil.go index 84482f4cc..e14bcf267 100644 --- a/compiler/astutil/astutil.go +++ b/compiler/astutil/astutil.go @@ -5,6 +5,7 @@ import ( "go/ast" "go/token" "go/types" + "regexp" ) func RemoveParens(e ast.Expr) ast.Expr { @@ -71,6 +72,103 @@ func FuncKey(d *ast.FuncDecl) string { return recv.(*ast.Ident).Name + "." + d.Name.Name } +// anyDocLine calls the given predicate on all associated documentation +// lines and line-comment lines from the given node. +// If the predicate returns true for any line then true is returned. +func anyDocLine(node any, predicate func(line string) bool) bool { + switch a := node.(type) { + case *ast.Comment: + return a != nil && predicate(a.Text) + case *ast.CommentGroup: + if a != nil { + for _, c := range a.List { + if anyDocLine(c, predicate) { + return true + } + } + } + return false + case *ast.Field: + return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) + case *ast.File: + return a != nil && anyDocLine(a.Doc, predicate) + case *ast.FuncDecl: + return a != nil && anyDocLine(a.Doc, predicate) + case *ast.GenDecl: + return a != nil && anyDocLine(a.Doc, predicate) + case *ast.ImportSpec: + return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) + case *ast.TypeSpec: + return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) + case *ast.ValueSpec: + return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) + default: + panic(fmt.Errorf(`unexpected node type to get doc from: %T`, node)) + } +} + +// directiveMatcher is a regex which matches a GopherJS directive +// and finds the directive action. +// +// This matches the largest directive action until whitespace or EOL +// to differentiate from any directive action which is a prefix +// for another directive action. +var directiveMatcher = regexp.MustCompile(`^\/(?:\/|\*)gopherjs:([\w-]+)`) + +// hasDirective returns true if the associated documentation +// or line comments for the given node have the given directive action. +// +// All gopherjs directives must start with `//gopherjs:` or `/*gopherjs:` +// and followed by an action without any whitespace. The action must be +// one or more letter, decimal, underscore, or hyphen. +// +// see https://pkg.go.dev/cmd/compile#hdr-Compiler_Directives +func hasDirective(node any, directive string) bool { + return anyDocLine(node, func(line string) bool { + m := directiveMatcher.FindStringSubmatch(line) + return len(m) == 2 && m[1] == directive + }) +} + +// PruneOriginal returns true if gopherjs:prune-original directive is present +// before a function decl. +// +// `//gopherjs:prune-original` is a GopherJS-specific directive, which can be +// applied to functions in native overlays and will instruct the augmentation +// logic to delete the body of a standard library function that was replaced. +// This directive can be used to remove code that would be invalid in GopherJS, +// such as code expecting ints to be 64-bit. It should be used with caution +// since it may create unused imports in the original source file. +func PruneOriginal(d *ast.FuncDecl) bool { + return hasDirective(d, `prune-original`) +} + +// KeepOriginal returns true if gopherjs:keep-original directive is present +// before a function decl. +// +// `//gopherjs:keep-original` is a GopherJS-specific directive, which can be +// applied to functions in native overlays and will instruct the augmentation +// logic to expose the original function such that it can be called. For a +// function in the original called `foo`, it will be accessible by the name +// `_gopherjs_original_foo`. +func KeepOriginal(d *ast.FuncDecl) bool { + return hasDirective(d, `keep-original`) +} + +// Purge returns true if gopherjs:purge directive is present +// on a struct, interface, variables, constants, and functions. +// +// `//gopherjs:purge` is a GopherJS-specific directive, which can be +// applied in native overlays and will instruct the augmentation logic to +// delete part of the standard library without a replacement. This directive +// can be used to remove code that would be invalid in GopherJS, such as code +// using unsupported features (e.g. generic interfaces and methods before +// generics were fully supported). It should be used with caution since it +// may remove needed dependencies. +func Purge(d any) bool { + return hasDirective(d, `purge`) +} + // FindLoopStmt tries to find the loop statement among the AST nodes in the // |stack| that corresponds to the break/continue statement represented by // branch. diff --git a/compiler/astutil/astutil_test.go b/compiler/astutil/astutil_test.go index f13ae0a1b..622b9f3ee 100644 --- a/compiler/astutil/astutil_test.go +++ b/compiler/astutil/astutil_test.go @@ -1,6 +1,8 @@ package astutil import ( + "fmt" + "go/ast" "go/token" "testing" @@ -82,6 +84,416 @@ func TestFuncKey(t *testing.T) { } } +func TestPruneOriginal(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: "no comment", + src: `package testpackage; + func foo() {}`, + want: false, + }, { + desc: "regular godoc", + src: `package testpackage; + // foo does something + func foo() {}`, + want: false, + }, { + desc: "only directive", + src: `package testpackage; + //gopherjs:prune-original + func foo() {}`, + want: true, + }, { + desc: "directive with explanation", + src: `package testpackage; + //gopherjs:prune-original because reasons + func foo() {}`, + want: true, + }, { + desc: "directive in godoc", + src: `package testpackage; + // foo does something + //gopherjs:prune-original + func foo() {}`, + want: true, + }, + } + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + fdecl := srctesting.ParseFuncDecl(t, test.src) + if got := PruneOriginal(fdecl); got != test.want { + t.Errorf("PruneOriginal() returned %t, want %t", got, test.want) + } + }) + } +} + +func TestHasDirectiveOnDecl(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: `no comment on function`, + src: `package testpackage; + func foo() {}`, + want: false, + }, { + desc: `no directive on function with comment`, + src: `package testpackage; + // foo has no directive + func foo() {}`, + want: false, + }, { + desc: `wrong directive on function`, + src: `package testpackage; + //gopherjs:wrong-directive + func foo() {}`, + want: false, + }, { + desc: `correct directive on function`, + src: `package testpackage; + //gopherjs:do-stuff + // foo has a directive to do stuff + func foo() {}`, + want: true, + }, { + desc: `correct directive in multiline comment on function`, + src: `package testpackage; + /*gopherjs:do-stuff + foo has a directive to do stuff + */ + func foo() {}`, + want: true, + }, { + desc: `invalid directive in multiline comment on function`, + src: `package testpackage; + /* + gopherjs:do-stuff + */ + func foo() {}`, + want: false, + }, { + desc: `prefix directive on function`, + src: `package testpackage; + //gopherjs:do-stuffs + func foo() {}`, + want: false, + }, { + desc: `multiple directives on function`, + src: `package testpackage; + //gopherjs:wrong-directive + //gopherjs:do-stuff + //gopherjs:another-directive + func foo() {}`, + want: true, + }, { + desc: `directive with explanation on function`, + src: `package testpackage; + //gopherjs:do-stuff 'cause we can + func foo() {}`, + want: true, + }, { + desc: `no directive on type declaration`, + src: `package testpackage; + // Foo has a comment + type Foo int`, + want: false, + }, { + desc: `directive on type declaration`, + src: `package testpackage; + //gopherjs:do-stuff + type Foo int`, + want: true, + }, { + desc: `no directive on const declaration`, + src: `package testpackage; + const foo = 42`, + want: false, + }, { + desc: `directive on const documentation`, + src: `package testpackage; + //gopherjs:do-stuff + const foo = 42`, + want: true, + }, { + desc: `no directive on var declaration`, + src: `package testpackage; + var foo = 42`, + want: false, + }, { + desc: `directive on var documentation`, + src: `package testpackage; + //gopherjs:do-stuff + var foo = 42`, + want: true, + }, { + desc: `no directive on var declaration`, + src: `package testpackage; + import _ "embed"`, + want: false, + }, { + desc: `directive on var documentation`, + src: `package testpackage; + //gopherjs:do-stuff + import _ "embed"`, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + decl := srctesting.ParseDecl(t, test.src) + if got := hasDirective(decl, action); got != test.want { + t.Errorf(`hasDirective(%T, %q) returned %t, want %t`, decl, action, got, test.want) + } + }) + } +} + +func TestHasDirectiveOnSpec(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: `no directive on type specification`, + src: `package testpackage; + type Foo int`, + want: false, + }, { + desc: `directive in doc on type specification`, + src: `package testpackage; + type ( + //gopherjs:do-stuff + Foo int + )`, + want: true, + }, { + desc: `directive in line on type specification`, + src: `package testpackage; + type Foo int //gopherjs:do-stuff`, + want: true, + }, { + desc: `no directive on const specification`, + src: `package testpackage; + const foo = 42`, + want: false, + }, { + desc: `directive in doc on const specification`, + src: `package testpackage; + const ( + //gopherjs:do-stuff + foo = 42 + )`, + want: true, + }, { + desc: `directive in line on const specification`, + src: `package testpackage; + const foo = 42 //gopherjs:do-stuff`, + want: true, + }, { + desc: `no directive on var specification`, + src: `package testpackage; + var foo = 42`, + want: false, + }, { + desc: `directive in doc on var specification`, + src: `package testpackage; + var ( + //gopherjs:do-stuff + foo = 42 + )`, + want: true, + }, { + desc: `directive in line on var specification`, + src: `package testpackage; + var foo = 42 //gopherjs:do-stuff`, + want: true, + }, { + desc: `no directive on import specification`, + src: `package testpackage; + import _ "embed"`, + want: false, + }, { + desc: `directive in doc on import specification`, + src: `package testpackage; + import ( + //gopherjs:do-stuff + _ "embed" + )`, + want: true, + }, { + desc: `directive in line on import specification`, + src: `package testpackage; + import _ "embed" //gopherjs:do-stuff`, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + spec := srctesting.ParseSpec(t, test.src) + if got := hasDirective(spec, action); got != test.want { + t.Errorf(`hasDirective(%T, %q) returned %t, want %t`, spec, action, got, test.want) + } + }) + } +} + +func TestHasDirectiveOnFile(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: `no directive on file`, + src: `package testpackage; + //gopherjs:do-stuff + type Foo int`, + want: false, + }, { + desc: `directive on file`, + src: `//gopherjs:do-stuff + package testpackage; + type Foo int`, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + fset := token.NewFileSet() + file := srctesting.Parse(t, fset, test.src) + if got := hasDirective(file, action); got != test.want { + t.Errorf(`hasDirective(%T, %q) returned %t, want %t`, file, action, got, test.want) + } + }) + } +} + +func TestHasDirectiveOnField(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: `no directive on struct field`, + src: `package testpackage; + type Foo struct { + bar int + }`, + want: false, + }, { + desc: `directive in doc on struct field`, + src: `package testpackage; + type Foo struct { + //gopherjs:do-stuff + bar int + }`, + want: true, + }, { + desc: `directive in line on struct field`, + src: `package testpackage; + type Foo struct { + bar int //gopherjs:do-stuff + }`, + want: true, + }, { + desc: `no directive on interface method`, + src: `package testpackage; + type Foo interface { + Bar(a int) int + }`, + want: false, + }, { + desc: `directive in doc on interface method`, + src: `package testpackage; + type Foo interface { + //gopherjs:do-stuff + Bar(a int) int + }`, + want: true, + }, { + desc: `directive in line on interface method`, + src: `package testpackage; + type Foo interface { + Bar(a int) int //gopherjs:do-stuff + }`, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + spec := srctesting.ParseSpec(t, test.src) + tspec := spec.(*ast.TypeSpec) + var field *ast.Field + switch typeNode := tspec.Type.(type) { + case *ast.StructType: + field = typeNode.Fields.List[0] + case *ast.InterfaceType: + field = typeNode.Methods.List[0] + default: + t.Errorf(`unexpected node type, %T, when finding field`, typeNode) + return + } + if got := hasDirective(field, action); got != test.want { + t.Errorf(`hasDirective(%T, %q) returned %t, want %t`, field, action, got, test.want) + } + }) + } +} + +func TestHasDirectiveBadCase(t *testing.T) { + tests := []struct { + desc string + node any + want string + }{ + { + desc: `untyped nil node`, + node: nil, + want: `unexpected node type to get doc from: `, + }, { + desc: `unexpected node type`, + node: &ast.ArrayType{}, + want: `unexpected node type to get doc from: *ast.ArrayType`, + }, { + desc: `nil expected node type`, + node: (*ast.FuncDecl)(nil), + want: ``, // no panic + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + + var got string + func() { + defer func() { got = fmt.Sprint(recover()) }() + hasDirective(test.node, action) + }() + if got != test.want { + t.Errorf(`hasDirective(%T, %q) returned %s, want %s`, test.node, action, got, test.want) + } + }) + } +} + func TestEndsWithReturn(t *testing.T) { tests := []struct { desc string diff --git a/compiler/astutil/directives.go b/compiler/astutil/directives.go deleted file mode 100644 index 088b0371c..000000000 --- a/compiler/astutil/directives.go +++ /dev/null @@ -1,104 +0,0 @@ -package astutil - -import ( - "fmt" - "go/ast" - "regexp" -) - -// PruneOriginal returns true if gopherjs:prune-original directive is present -// before a function decl. -// -// `//gopherjs:prune-original` is a GopherJS-specific directive, which can be -// applied to functions in native overlays and will instruct the augmentation -// logic to delete the body of a standard library function that was replaced. -// This directive can be used to remove code that would be invalid in GopherJS, -// such as code expecting ints to be 64-bit. It should be used with caution -// since it may create unused imports in the original source file. -func PruneOriginal(d *ast.FuncDecl) bool { - return hasDirective(d, `prune-original`) -} - -// KeepOriginal returns true if gopherjs:keep-original directive is present -// before a function decl. -// -// `//gopherjs:keep-original` is a GopherJS-specific directive, which can be -// applied to functions in native overlays and will instruct the augmentation -// logic to expose the original function such that it can be called. For a -// function in the original called `foo`, it will be accessible by the name -// `_gopherjs_original_foo`. -func KeepOriginal(d *ast.FuncDecl) bool { - return hasDirective(d, `keep-original`) -} - -// PurgeOriginal returns true if gopherjs:purge-original directive is present -// on a struct, interface, variables, constants, and functions. -// -// `//gopherjs:purge-original` is a GopherJS-specific directive, which can be -// applied in native overlays and will instruct the augmentation logic to -// delete part of the standard library without a replacement. This directive -// can be used to remove code that would be invalid in GopherJS, such as code -// using unsupported features (e.g. generic interfaces and methods before -// generics were fully supported). -// It should be used with caution since it may remove needed dependencies. -func PurgeOriginal(d any) bool { - return hasDirective(d, `purge-original`) -} - -// anyDocLine calls the given predicate on all associated documentation -// lines and line-comment lines from the given node. -// If the predicate returns true for any line then true is returned. -func anyDocLine(node any, predicate func(line string) bool) bool { - switch a := node.(type) { - case *ast.Comment: - return a != nil && predicate(a.Text) - case *ast.CommentGroup: - if a != nil { - for _, c := range a.List { - if anyDocLine(c, predicate) { - return true - } - } - } - return false - case *ast.Field: - return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) - case *ast.File: - return a != nil && anyDocLine(a.Doc, predicate) - case *ast.FuncDecl: - return a != nil && anyDocLine(a.Doc, predicate) - case *ast.GenDecl: - return a != nil && anyDocLine(a.Doc, predicate) - case *ast.ImportSpec: - return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) - case *ast.TypeSpec: - return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) - case *ast.ValueSpec: - return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) - default: - panic(fmt.Errorf(`unexpected node type to get doc from: %T`, node)) - } -} - -// directiveMatcher is a regex which matches a GopherJS directive -// and finds the directive action. -// -// This matches the largest directive action until whitespace or EOL -// to differentiate from any directive action which is a prefix -// for another directive action. -var directiveMatcher = regexp.MustCompile(`^\/(?:\/|\*)gopherjs:([\w-]+)`) - -// hasDirective returns true if the associated documentation -// or line comments for the given node have the given directive action. -// -// All gopherjs directives must start with `//gopherjs:` or `/*gopherjs:` -// and followed by an action without any whitespace. The action must be -// one or more letter, decimal, underscore, or hyphen. -// -// see https://pkg.go.dev/cmd/compile#hdr-Compiler_Directives -func hasDirective(node any, directive string) bool { - return anyDocLine(node, func(line string) bool { - m := directiveMatcher.FindStringSubmatch(line) - return len(m) == 2 && m[1] == directive - }) -} diff --git a/compiler/astutil/directives_test.go b/compiler/astutil/directives_test.go deleted file mode 100644 index 9325f73f4..000000000 --- a/compiler/astutil/directives_test.go +++ /dev/null @@ -1,420 +0,0 @@ -package astutil - -import ( - "fmt" - "go/ast" - "go/token" - "testing" - - "github.com/gopherjs/gopherjs/internal/srctesting" -) - -func TestPruneOriginal(t *testing.T) { - tests := []struct { - desc string - src string - want bool - }{ - { - desc: "no comment", - src: `package testpackage; - func foo() {}`, - want: false, - }, { - desc: "regular godoc", - src: `package testpackage; - // foo does something - func foo() {}`, - want: false, - }, { - desc: "only directive", - src: `package testpackage; - //gopherjs:prune-original - func foo() {}`, - want: true, - }, { - desc: "directive with explanation", - src: `package testpackage; - //gopherjs:prune-original because reasons - func foo() {}`, - want: true, - }, { - desc: "directive in godoc", - src: `package testpackage; - // foo does something - //gopherjs:prune-original - func foo() {}`, - want: true, - }, - } - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - fdecl := srctesting.ParseFuncDecl(t, test.src) - if got := PruneOriginal(fdecl); got != test.want { - t.Errorf("PruneOriginal() returned %t, want %t", got, test.want) - } - }) - } -} - -func TestHasDirectiveOnDecl(t *testing.T) { - tests := []struct { - desc string - src string - want bool - }{ - { - desc: `no comment on function`, - src: `package testpackage; - func foo() {}`, - want: false, - }, { - desc: `no directive on function with comment`, - src: `package testpackage; - // foo has no directive - func foo() {}`, - want: false, - }, { - desc: `wrong directive on function`, - src: `package testpackage; - //gopherjs:wrong-directive - func foo() {}`, - want: false, - }, { - desc: `correct directive on function`, - src: `package testpackage; - //gopherjs:do-stuff - // foo has a directive to do stuff - func foo() {}`, - want: true, - }, { - desc: `correct directive in multiline comment on function`, - src: `package testpackage; - /*gopherjs:do-stuff - foo has a directive to do stuff - */ - func foo() {}`, - want: true, - }, { - desc: `invalid directive in multiline comment on function`, - src: `package testpackage; - /* - gopherjs:do-stuff - */ - func foo() {}`, - want: false, - }, { - desc: `prefix directive on function`, - src: `package testpackage; - //gopherjs:do-stuffs - func foo() {}`, - want: false, - }, { - desc: `multiple directives on function`, - src: `package testpackage; - //gopherjs:wrong-directive - //gopherjs:do-stuff - //gopherjs:another-directive - func foo() {}`, - want: true, - }, { - desc: `directive with explanation on function`, - src: `package testpackage; - //gopherjs:do-stuff 'cause we can - func foo() {}`, - want: true, - }, { - desc: `no directive on type declaration`, - src: `package testpackage; - // Foo has a comment - type Foo int`, - want: false, - }, { - desc: `directive on type declaration`, - src: `package testpackage; - //gopherjs:do-stuff - type Foo int`, - want: true, - }, { - desc: `no directive on const declaration`, - src: `package testpackage; - const foo = 42`, - want: false, - }, { - desc: `directive on const documentation`, - src: `package testpackage; - //gopherjs:do-stuff - const foo = 42`, - want: true, - }, { - desc: `no directive on var declaration`, - src: `package testpackage; - var foo = 42`, - want: false, - }, { - desc: `directive on var documentation`, - src: `package testpackage; - //gopherjs:do-stuff - var foo = 42`, - want: true, - }, { - desc: `no directive on var declaration`, - src: `package testpackage; - import _ "embed"`, - want: false, - }, { - desc: `directive on var documentation`, - src: `package testpackage; - //gopherjs:do-stuff - import _ "embed"`, - want: true, - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - const action = `do-stuff` - decl := srctesting.ParseDecl(t, test.src) - if got := hasDirective(decl, action); got != test.want { - t.Errorf(`hasDirective(%T, %q) returned %t, want %t`, decl, action, got, test.want) - } - }) - } -} - -func TestHasDirectiveOnSpec(t *testing.T) { - tests := []struct { - desc string - src string - want bool - }{ - { - desc: `no directive on type specification`, - src: `package testpackage; - type Foo int`, - want: false, - }, { - desc: `directive in doc on type specification`, - src: `package testpackage; - type ( - //gopherjs:do-stuff - Foo int - )`, - want: true, - }, { - desc: `directive in line on type specification`, - src: `package testpackage; - type Foo int //gopherjs:do-stuff`, - want: true, - }, { - desc: `no directive on const specification`, - src: `package testpackage; - const foo = 42`, - want: false, - }, { - desc: `directive in doc on const specification`, - src: `package testpackage; - const ( - //gopherjs:do-stuff - foo = 42 - )`, - want: true, - }, { - desc: `directive in line on const specification`, - src: `package testpackage; - const foo = 42 //gopherjs:do-stuff`, - want: true, - }, { - desc: `no directive on var specification`, - src: `package testpackage; - var foo = 42`, - want: false, - }, { - desc: `directive in doc on var specification`, - src: `package testpackage; - var ( - //gopherjs:do-stuff - foo = 42 - )`, - want: true, - }, { - desc: `directive in line on var specification`, - src: `package testpackage; - var foo = 42 //gopherjs:do-stuff`, - want: true, - }, { - desc: `no directive on import specification`, - src: `package testpackage; - import _ "embed"`, - want: false, - }, { - desc: `directive in doc on import specification`, - src: `package testpackage; - import ( - //gopherjs:do-stuff - _ "embed" - )`, - want: true, - }, { - desc: `directive in line on import specification`, - src: `package testpackage; - import _ "embed" //gopherjs:do-stuff`, - want: true, - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - const action = `do-stuff` - spec := srctesting.ParseSpec(t, test.src) - if got := hasDirective(spec, action); got != test.want { - t.Errorf(`hasDirective(%T, %q) returned %t, want %t`, spec, action, got, test.want) - } - }) - } -} - -func TestHasDirectiveOnFile(t *testing.T) { - tests := []struct { - desc string - src string - want bool - }{ - { - desc: `no directive on file`, - src: `package testpackage; - //gopherjs:do-stuff - type Foo int`, - want: false, - }, { - desc: `directive on file`, - src: `//gopherjs:do-stuff - package testpackage; - type Foo int`, - want: true, - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - const action = `do-stuff` - fset := token.NewFileSet() - file := srctesting.Parse(t, fset, test.src) - if got := hasDirective(file, action); got != test.want { - t.Errorf(`hasDirective(%T, %q) returned %t, want %t`, file, action, got, test.want) - } - }) - } -} - -func TestHasDirectiveOnField(t *testing.T) { - tests := []struct { - desc string - src string - want bool - }{ - { - desc: `no directive on struct field`, - src: `package testpackage; - type Foo struct { - bar int - }`, - want: false, - }, { - desc: `directive in doc on struct field`, - src: `package testpackage; - type Foo struct { - //gopherjs:do-stuff - bar int - }`, - want: true, - }, { - desc: `directive in line on struct field`, - src: `package testpackage; - type Foo struct { - bar int //gopherjs:do-stuff - }`, - want: true, - }, { - desc: `no directive on interface method`, - src: `package testpackage; - type Foo interface { - Bar(a int) int - }`, - want: false, - }, { - desc: `directive in doc on interface method`, - src: `package testpackage; - type Foo interface { - //gopherjs:do-stuff - Bar(a int) int - }`, - want: true, - }, { - desc: `directive in line on interface method`, - src: `package testpackage; - type Foo interface { - Bar(a int) int //gopherjs:do-stuff - }`, - want: true, - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - const action = `do-stuff` - spec := srctesting.ParseSpec(t, test.src) - tspec := spec.(*ast.TypeSpec) - var field *ast.Field - switch typeNode := tspec.Type.(type) { - case *ast.StructType: - field = typeNode.Fields.List[0] - case *ast.InterfaceType: - field = typeNode.Methods.List[0] - default: - t.Errorf(`unexpected node type, %T, when finding field`, typeNode) - return - } - if got := hasDirective(field, action); got != test.want { - t.Errorf(`hasDirective(%T, %q) returned %t, want %t`, field, action, got, test.want) - } - }) - } -} - -func TestHasDirectiveBadCase(t *testing.T) { - tests := []struct { - desc string - node any - want string - }{ - { - desc: `untyped nil node`, - node: nil, - want: `unexpected node type to get doc from: `, - }, { - desc: `unexpected node type`, - node: &ast.ArrayType{}, - want: `unexpected node type to get doc from: *ast.ArrayType`, - }, { - desc: `nil expected node type`, - node: (*ast.FuncDecl)(nil), - want: ``, // no panic - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - const action = `do-stuff` - - var got string - func() { - defer func() { got = fmt.Sprint(recover()) }() - hasDirective(test.node, action) - }() - if got != test.want { - t.Errorf(`hasDirective(%T, %q) returned %s, want %s`, test.node, action, got, test.want) - } - }) - } -} From f0920104befa56de8fb545288ec0fe443a4cb310 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Wed, 6 Dec 2023 14:40:30 -0700 Subject: [PATCH 04/11] Added directives --- internal/srctesting/srctesting.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/internal/srctesting/srctesting.go b/internal/srctesting/srctesting.go index c9ff63575..3a1f418f3 100644 --- a/internal/srctesting/srctesting.go +++ b/internal/srctesting/srctesting.go @@ -62,20 +62,6 @@ func ParseDecl(t *testing.T, src string) ast.Decl { return file.Decls[0] } -// ParseFuncDecl parses source with a single function defined and returns the -// function AST. -// -// Fails the test if there isn't exactly one function declared in the source. -func ParseFuncDecl(t *testing.T, src string) *ast.FuncDecl { - t.Helper() - decl := ParseDecl(t, src) - fdecl, ok := decl.(*ast.FuncDecl) - if !ok { - t.Fatalf("Got %T decl, expected *ast.FuncDecl", decl) - } - return fdecl -} - // ParseSpec parses source with a single declaration containing // a single specification and returns that specification AST. // @@ -94,6 +80,20 @@ func ParseSpec(t *testing.T, src string) ast.Spec { return gdecl.Specs[0] } +// ParseFuncDecl parses source with a single function defined and returns the +// function AST. +// +// Fails the test if there isn't exactly one function declared in the source. +func ParseFuncDecl(t *testing.T, src string) *ast.FuncDecl { + t.Helper() + decl := ParseDecl(t, src) + fdecl, ok := decl.(*ast.FuncDecl) + if !ok { + t.Fatalf("Got %T decl, expected *ast.FuncDecl", decl) + } + return fdecl +} + // Format AST node into a string. // // The node type must be *ast.File, *printer.CommentedNode, []ast.Decl, From 94c469eb6a58063f995b30e8c0edb1116c738b42 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Wed, 6 Dec 2023 14:41:06 -0700 Subject: [PATCH 05/11] Added directives --- internal/srctesting/srctesting.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/internal/srctesting/srctesting.go b/internal/srctesting/srctesting.go index 3a1f418f3..4e374845e 100644 --- a/internal/srctesting/srctesting.go +++ b/internal/srctesting/srctesting.go @@ -48,6 +48,20 @@ func Check(t *testing.T, fset *token.FileSet, files ...*ast.File) (*types.Info, return typesInfo, typesPkg } +// ParseFuncDecl parses source with a single function defined and returns the +// function AST. +// +// Fails the test if there isn't exactly one function declared in the source. +func ParseFuncDecl(t *testing.T, src string) *ast.FuncDecl { + t.Helper() + decl := ParseDecl(t, src) + fdecl, ok := decl.(*ast.FuncDecl) + if !ok { + t.Fatalf("Got %T decl, expected *ast.FuncDecl", decl) + } + return fdecl +} + // ParseDecl parses source with a single declaration and // returns that declaration AST. // @@ -80,20 +94,6 @@ func ParseSpec(t *testing.T, src string) ast.Spec { return gdecl.Specs[0] } -// ParseFuncDecl parses source with a single function defined and returns the -// function AST. -// -// Fails the test if there isn't exactly one function declared in the source. -func ParseFuncDecl(t *testing.T, src string) *ast.FuncDecl { - t.Helper() - decl := ParseDecl(t, src) - fdecl, ok := decl.(*ast.FuncDecl) - if !ok { - t.Fatalf("Got %T decl, expected *ast.FuncDecl", decl) - } - return fdecl -} - // Format AST node into a string. // // The node type must be *ast.File, *printer.CommentedNode, []ast.Decl, From 4eed3b72e94b4cedb054256d5cdb4e4b9eb1c1a9 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Tue, 5 Dec 2023 15:59:55 -0700 Subject: [PATCH 06/11] Adding new directives --- compiler/astutil/astutil.go | 94 ++++++-- compiler/astutil/astutil_test.go | 364 ++++++++++++++++++++++++++++++ internal/srctesting/srctesting.go | 36 ++- 3 files changed, 471 insertions(+), 23 deletions(-) diff --git a/compiler/astutil/astutil.go b/compiler/astutil/astutil.go index 30febe1cb..e14bcf267 100644 --- a/compiler/astutil/astutil.go +++ b/compiler/astutil/astutil.go @@ -5,7 +5,7 @@ import ( "go/ast" "go/token" "go/types" - "strings" + "regexp" ) func RemoveParens(e ast.Expr) ast.Expr { @@ -72,6 +72,64 @@ func FuncKey(d *ast.FuncDecl) string { return recv.(*ast.Ident).Name + "." + d.Name.Name } +// anyDocLine calls the given predicate on all associated documentation +// lines and line-comment lines from the given node. +// If the predicate returns true for any line then true is returned. +func anyDocLine(node any, predicate func(line string) bool) bool { + switch a := node.(type) { + case *ast.Comment: + return a != nil && predicate(a.Text) + case *ast.CommentGroup: + if a != nil { + for _, c := range a.List { + if anyDocLine(c, predicate) { + return true + } + } + } + return false + case *ast.Field: + return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) + case *ast.File: + return a != nil && anyDocLine(a.Doc, predicate) + case *ast.FuncDecl: + return a != nil && anyDocLine(a.Doc, predicate) + case *ast.GenDecl: + return a != nil && anyDocLine(a.Doc, predicate) + case *ast.ImportSpec: + return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) + case *ast.TypeSpec: + return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) + case *ast.ValueSpec: + return a != nil && (anyDocLine(a.Doc, predicate) || anyDocLine(a.Comment, predicate)) + default: + panic(fmt.Errorf(`unexpected node type to get doc from: %T`, node)) + } +} + +// directiveMatcher is a regex which matches a GopherJS directive +// and finds the directive action. +// +// This matches the largest directive action until whitespace or EOL +// to differentiate from any directive action which is a prefix +// for another directive action. +var directiveMatcher = regexp.MustCompile(`^\/(?:\/|\*)gopherjs:([\w-]+)`) + +// hasDirective returns true if the associated documentation +// or line comments for the given node have the given directive action. +// +// All gopherjs directives must start with `//gopherjs:` or `/*gopherjs:` +// and followed by an action without any whitespace. The action must be +// one or more letter, decimal, underscore, or hyphen. +// +// see https://pkg.go.dev/cmd/compile#hdr-Compiler_Directives +func hasDirective(node any, directive string) bool { + return anyDocLine(node, func(line string) bool { + m := directiveMatcher.FindStringSubmatch(line) + return len(m) == 2 && m[1] == directive + }) +} + // PruneOriginal returns true if gopherjs:prune-original directive is present // before a function decl. // @@ -82,15 +140,7 @@ func FuncKey(d *ast.FuncDecl) string { // such as code expecting ints to be 64-bit. It should be used with caution // since it may create unused imports in the original source file. func PruneOriginal(d *ast.FuncDecl) bool { - if d.Doc == nil { - return false - } - for _, c := range d.Doc.List { - if strings.HasPrefix(c.Text, "//gopherjs:prune-original") { - return true - } - } - return false + return hasDirective(d, `prune-original`) } // KeepOriginal returns true if gopherjs:keep-original directive is present @@ -102,15 +152,21 @@ func PruneOriginal(d *ast.FuncDecl) bool { // function in the original called `foo`, it will be accessible by the name // `_gopherjs_original_foo`. func KeepOriginal(d *ast.FuncDecl) bool { - if d.Doc == nil { - return false - } - for _, c := range d.Doc.List { - if strings.HasPrefix(c.Text, "//gopherjs:keep-original") { - return true - } - } - return false + return hasDirective(d, `keep-original`) +} + +// Purge returns true if gopherjs:purge directive is present +// on a struct, interface, variables, constants, and functions. +// +// `//gopherjs:purge` is a GopherJS-specific directive, which can be +// applied in native overlays and will instruct the augmentation logic to +// delete part of the standard library without a replacement. This directive +// can be used to remove code that would be invalid in GopherJS, such as code +// using unsupported features (e.g. generic interfaces and methods before +// generics were fully supported). It should be used with caution since it +// may remove needed dependencies. +func Purge(d any) bool { + return hasDirective(d, `purge`) } // FindLoopStmt tries to find the loop statement among the AST nodes in the diff --git a/compiler/astutil/astutil_test.go b/compiler/astutil/astutil_test.go index a996ae73f..622b9f3ee 100644 --- a/compiler/astutil/astutil_test.go +++ b/compiler/astutil/astutil_test.go @@ -1,6 +1,8 @@ package astutil import ( + "fmt" + "go/ast" "go/token" "testing" @@ -130,6 +132,368 @@ func TestPruneOriginal(t *testing.T) { } } +func TestHasDirectiveOnDecl(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: `no comment on function`, + src: `package testpackage; + func foo() {}`, + want: false, + }, { + desc: `no directive on function with comment`, + src: `package testpackage; + // foo has no directive + func foo() {}`, + want: false, + }, { + desc: `wrong directive on function`, + src: `package testpackage; + //gopherjs:wrong-directive + func foo() {}`, + want: false, + }, { + desc: `correct directive on function`, + src: `package testpackage; + //gopherjs:do-stuff + // foo has a directive to do stuff + func foo() {}`, + want: true, + }, { + desc: `correct directive in multiline comment on function`, + src: `package testpackage; + /*gopherjs:do-stuff + foo has a directive to do stuff + */ + func foo() {}`, + want: true, + }, { + desc: `invalid directive in multiline comment on function`, + src: `package testpackage; + /* + gopherjs:do-stuff + */ + func foo() {}`, + want: false, + }, { + desc: `prefix directive on function`, + src: `package testpackage; + //gopherjs:do-stuffs + func foo() {}`, + want: false, + }, { + desc: `multiple directives on function`, + src: `package testpackage; + //gopherjs:wrong-directive + //gopherjs:do-stuff + //gopherjs:another-directive + func foo() {}`, + want: true, + }, { + desc: `directive with explanation on function`, + src: `package testpackage; + //gopherjs:do-stuff 'cause we can + func foo() {}`, + want: true, + }, { + desc: `no directive on type declaration`, + src: `package testpackage; + // Foo has a comment + type Foo int`, + want: false, + }, { + desc: `directive on type declaration`, + src: `package testpackage; + //gopherjs:do-stuff + type Foo int`, + want: true, + }, { + desc: `no directive on const declaration`, + src: `package testpackage; + const foo = 42`, + want: false, + }, { + desc: `directive on const documentation`, + src: `package testpackage; + //gopherjs:do-stuff + const foo = 42`, + want: true, + }, { + desc: `no directive on var declaration`, + src: `package testpackage; + var foo = 42`, + want: false, + }, { + desc: `directive on var documentation`, + src: `package testpackage; + //gopherjs:do-stuff + var foo = 42`, + want: true, + }, { + desc: `no directive on var declaration`, + src: `package testpackage; + import _ "embed"`, + want: false, + }, { + desc: `directive on var documentation`, + src: `package testpackage; + //gopherjs:do-stuff + import _ "embed"`, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + decl := srctesting.ParseDecl(t, test.src) + if got := hasDirective(decl, action); got != test.want { + t.Errorf(`hasDirective(%T, %q) returned %t, want %t`, decl, action, got, test.want) + } + }) + } +} + +func TestHasDirectiveOnSpec(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: `no directive on type specification`, + src: `package testpackage; + type Foo int`, + want: false, + }, { + desc: `directive in doc on type specification`, + src: `package testpackage; + type ( + //gopherjs:do-stuff + Foo int + )`, + want: true, + }, { + desc: `directive in line on type specification`, + src: `package testpackage; + type Foo int //gopherjs:do-stuff`, + want: true, + }, { + desc: `no directive on const specification`, + src: `package testpackage; + const foo = 42`, + want: false, + }, { + desc: `directive in doc on const specification`, + src: `package testpackage; + const ( + //gopherjs:do-stuff + foo = 42 + )`, + want: true, + }, { + desc: `directive in line on const specification`, + src: `package testpackage; + const foo = 42 //gopherjs:do-stuff`, + want: true, + }, { + desc: `no directive on var specification`, + src: `package testpackage; + var foo = 42`, + want: false, + }, { + desc: `directive in doc on var specification`, + src: `package testpackage; + var ( + //gopherjs:do-stuff + foo = 42 + )`, + want: true, + }, { + desc: `directive in line on var specification`, + src: `package testpackage; + var foo = 42 //gopherjs:do-stuff`, + want: true, + }, { + desc: `no directive on import specification`, + src: `package testpackage; + import _ "embed"`, + want: false, + }, { + desc: `directive in doc on import specification`, + src: `package testpackage; + import ( + //gopherjs:do-stuff + _ "embed" + )`, + want: true, + }, { + desc: `directive in line on import specification`, + src: `package testpackage; + import _ "embed" //gopherjs:do-stuff`, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + spec := srctesting.ParseSpec(t, test.src) + if got := hasDirective(spec, action); got != test.want { + t.Errorf(`hasDirective(%T, %q) returned %t, want %t`, spec, action, got, test.want) + } + }) + } +} + +func TestHasDirectiveOnFile(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: `no directive on file`, + src: `package testpackage; + //gopherjs:do-stuff + type Foo int`, + want: false, + }, { + desc: `directive on file`, + src: `//gopherjs:do-stuff + package testpackage; + type Foo int`, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + fset := token.NewFileSet() + file := srctesting.Parse(t, fset, test.src) + if got := hasDirective(file, action); got != test.want { + t.Errorf(`hasDirective(%T, %q) returned %t, want %t`, file, action, got, test.want) + } + }) + } +} + +func TestHasDirectiveOnField(t *testing.T) { + tests := []struct { + desc string + src string + want bool + }{ + { + desc: `no directive on struct field`, + src: `package testpackage; + type Foo struct { + bar int + }`, + want: false, + }, { + desc: `directive in doc on struct field`, + src: `package testpackage; + type Foo struct { + //gopherjs:do-stuff + bar int + }`, + want: true, + }, { + desc: `directive in line on struct field`, + src: `package testpackage; + type Foo struct { + bar int //gopherjs:do-stuff + }`, + want: true, + }, { + desc: `no directive on interface method`, + src: `package testpackage; + type Foo interface { + Bar(a int) int + }`, + want: false, + }, { + desc: `directive in doc on interface method`, + src: `package testpackage; + type Foo interface { + //gopherjs:do-stuff + Bar(a int) int + }`, + want: true, + }, { + desc: `directive in line on interface method`, + src: `package testpackage; + type Foo interface { + Bar(a int) int //gopherjs:do-stuff + }`, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + spec := srctesting.ParseSpec(t, test.src) + tspec := spec.(*ast.TypeSpec) + var field *ast.Field + switch typeNode := tspec.Type.(type) { + case *ast.StructType: + field = typeNode.Fields.List[0] + case *ast.InterfaceType: + field = typeNode.Methods.List[0] + default: + t.Errorf(`unexpected node type, %T, when finding field`, typeNode) + return + } + if got := hasDirective(field, action); got != test.want { + t.Errorf(`hasDirective(%T, %q) returned %t, want %t`, field, action, got, test.want) + } + }) + } +} + +func TestHasDirectiveBadCase(t *testing.T) { + tests := []struct { + desc string + node any + want string + }{ + { + desc: `untyped nil node`, + node: nil, + want: `unexpected node type to get doc from: `, + }, { + desc: `unexpected node type`, + node: &ast.ArrayType{}, + want: `unexpected node type to get doc from: *ast.ArrayType`, + }, { + desc: `nil expected node type`, + node: (*ast.FuncDecl)(nil), + want: ``, // no panic + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + const action = `do-stuff` + + var got string + func() { + defer func() { got = fmt.Sprint(recover()) }() + hasDirective(test.node, action) + }() + if got != test.want { + t.Errorf(`hasDirective(%T, %q) returned %s, want %s`, test.node, action, got, test.want) + } + }) + } +} + func TestEndsWithReturn(t *testing.T) { tests := []struct { desc string diff --git a/internal/srctesting/srctesting.go b/internal/srctesting/srctesting.go index 1d9cecd20..4e374845e 100644 --- a/internal/srctesting/srctesting.go +++ b/internal/srctesting/srctesting.go @@ -53,17 +53,45 @@ func Check(t *testing.T, fset *token.FileSet, files ...*ast.File) (*types.Info, // // Fails the test if there isn't exactly one function declared in the source. func ParseFuncDecl(t *testing.T, src string) *ast.FuncDecl { + t.Helper() + decl := ParseDecl(t, src) + fdecl, ok := decl.(*ast.FuncDecl) + if !ok { + t.Fatalf("Got %T decl, expected *ast.FuncDecl", decl) + } + return fdecl +} + +// ParseDecl parses source with a single declaration and +// returns that declaration AST. +// +// Fails the test if there isn't exactly one declaration in the source. +func ParseDecl(t *testing.T, src string) ast.Decl { t.Helper() fset := token.NewFileSet() file := Parse(t, fset, src) if l := len(file.Decls); l != 1 { - t.Fatalf("Got %d decls in the sources, expected exactly 1", l) + t.Fatalf(`Got %d decls in the sources, expected exactly 1`, l) } - fdecl, ok := file.Decls[0].(*ast.FuncDecl) + return file.Decls[0] +} + +// ParseSpec parses source with a single declaration containing +// a single specification and returns that specification AST. +// +// Fails the test if there isn't exactly one declaration and +// one specification in the source. +func ParseSpec(t *testing.T, src string) ast.Spec { + t.Helper() + decl := ParseDecl(t, src) + gdecl, ok := decl.(*ast.GenDecl) if !ok { - t.Fatalf("Got %T decl, expected *ast.FuncDecl", file.Decls[0]) + t.Fatalf("Got %T decl, expected *ast.GenDecl", decl) } - return fdecl + if l := len(gdecl.Specs); l != 1 { + t.Fatalf(`Got %d spec in the sources, expected exactly 1`, l) + } + return gdecl.Specs[0] } // Format AST node into a string. From 2693ad90c12ac8a726aeac4afd6ba5cc6c6ec35b Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Wed, 6 Dec 2023 17:41:14 -0700 Subject: [PATCH 07/11] Updating parseAndAugment to use new directives --- build/build.go | 275 ++++++++++++++++++-------- compiler/astutil/astutil.go | 127 ++++++++++-- compiler/astutil/astutil_test.go | 98 +++++++-- compiler/natives/src/net/http/http.go | 6 +- 4 files changed, 391 insertions(+), 115 deletions(-) diff --git a/build/build.go b/build/build.go index 070d05df1..6e5c450ab 100644 --- a/build/build.go +++ b/build/build.go @@ -117,6 +117,22 @@ func ImportDir(dir string, mode build.ImportMode, installSuffix string, buildTag return pkg, nil } +// overrideInfo is used by parseAndAugment methods to manage +// directives and how the overlay and original are merged. +type overrideInfo struct { + // KeepOriginal indicates that the original code should be kept + // but the identifier will be prefixed by `_gopherjs_original_foo`. + keepOriginal bool + + // pruneFuncBody indicates that the body of the function should + // be removed to prevent statements that are invalid in GopherJS. + pruneFuncBody bool + + // purgeMethods indicates that this info is for a struct and + // if a method has this struct as a receiver should also be removed. + purgeMethods bool +} + // parseAndAugment parses and returns all .go files of given pkg. // Standard Go library packages are augmented with files in compiler/natives folder. // If isTest is true and pkg.ImportPath has no _test suffix, package is built for running internal tests. @@ -125,31 +141,55 @@ func ImportDir(dir string, mode build.ImportMode, installSuffix string, buildTag // The native packages are augmented by the contents of natives.FS in the following way. // The file names do not matter except the usual `_test` suffix. The files for // native overrides get added to the package (even if they have the same name -// as an existing file from the standard library). For function identifiers that exist -// in the original AND the overrides AND that include the following directive in their comment: -// //gopherjs:keep-original, the original identifier in the AST gets prefixed by -// `_gopherjs_original_`. For other identifiers that exist in the original AND the overrides, -// the original identifier gets replaced by `_`. New identifiers that don't exist in original -// package get added. +// as an existing file from the standard library). +// +// - For function identifiers that exist in the original and the overrides and have the +// directive `gopherjs:keep-original`, the original identifier in the AST gets +// prefixed by `_gopherjs_original_`. +// - For functions that exist in the original and the overrides and have the +// directive `gopherjs:prune-original`, the original function body in the AST +// is removed. This removes code that is invalid in GopherJS. +// - For identifiers that exist in the original and the overrides +// and have the directive `gopherjs:purge`, both the original and override are removed. +// - Otherwise for identifiers that exist in the original and the overrides, +// the original identifier is removed. +// - New identifiers that don't exist in original package get added. func parseAndAugment(xctx XContext, pkg *PackageData, isTest bool, fileSet *token.FileSet) ([]*ast.File, []JSFile, error) { - var files []*ast.File + jsFiles, overlayFiles := parseOverlayFiles(xctx, pkg, isTest, fileSet) - type overrideInfo struct { - keepOriginal bool - pruneOriginal bool + originalFiles, err := parserOriginalFiles(pkg, fileSet) + if err != nil { + return nil, nil, err } + replacedDeclNames := make(map[string]overrideInfo) + for _, file := range overlayFiles { + augmentOverlayFile(file, replacedDeclNames) + pruneImports(file) + } + delete(replacedDeclNames, "init") + for _, file := range originalFiles { + augmentOriginalImports(pkg.ImportPath, file) + augmentOriginalFile(file, replacedDeclNames) + pruneImports(file) + } + + return append(overlayFiles, originalFiles...), jsFiles, nil +} + +// parseOverlayFiles loads and parses overlay files +// to augment the original files with. +func parseOverlayFiles(xctx XContext, pkg *PackageData, isTest bool, fileSet *token.FileSet) ([]JSFile, []*ast.File) { isXTest := strings.HasSuffix(pkg.ImportPath, "_test") importPath := pkg.ImportPath if isXTest { importPath = importPath[:len(importPath)-5] } - jsFiles := []JSFile{} - + var jsFiles []JSFile + var files []*ast.File nativesContext := overlayCtx(xctx.Env()) - if nativesPkg, err := nativesContext.Import(importPath, "", 0); err == nil { jsFiles = nativesPkg.JSFiles names := nativesPkg.GoFiles @@ -159,6 +199,7 @@ func parseAndAugment(xctx XContext, pkg *PackageData, isTest bool, fileSet *toke if isXTest { names = nativesPkg.XTestGoFiles } + for _, name := range names { fullPath := path.Join(nativesPkg.Dir, name) r, err := nativesContext.bctx.OpenFile(fullPath) @@ -173,34 +214,17 @@ func parseAndAugment(xctx XContext, pkg *PackageData, isTest bool, fileSet *toke panic(err) } r.Close() - for _, decl := range file.Decls { - switch d := decl.(type) { - case *ast.FuncDecl: - k := astutil.FuncKey(d) - replacedDeclNames[k] = overrideInfo{ - keepOriginal: astutil.KeepOriginal(d), - pruneOriginal: astutil.PruneOriginal(d), - } - case *ast.GenDecl: - switch d.Tok { - case token.TYPE: - for _, spec := range d.Specs { - replacedDeclNames[spec.(*ast.TypeSpec).Name.Name] = overrideInfo{} - } - case token.VAR, token.CONST: - for _, spec := range d.Specs { - for _, name := range spec.(*ast.ValueSpec).Names { - replacedDeclNames[name.Name] = overrideInfo{} - } - } - } - } - } + files = append(files, file) } } - delete(replacedDeclNames, "init") + return jsFiles, files +} + +// parserOriginalFiles loads and parses the original files to augment. +func parserOriginalFiles(pkg *PackageData, fileSet *token.FileSet) ([]*ast.File, error) { + var files []*ast.File var errList compiler.ErrorList for _, name := range pkg.GoFiles { if !filepath.IsAbs(name) { // name might be absolute if specified directly. E.g., `gopherjs build /abs/file.go`. @@ -208,7 +232,7 @@ func parseAndAugment(xctx XContext, pkg *PackageData, isTest bool, fileSet *toke } r, err := buildutil.OpenFile(pkg.bctx, name) if err != nil { - return nil, nil, err + return nil, err } file, err := parser.ParseFile(fileSet, name, r, parser.ParseComments) r.Close() @@ -226,68 +250,155 @@ func parseAndAugment(xctx XContext, pkg *PackageData, isTest bool, fileSet *toke continue } - switch pkg.ImportPath { - case "crypto/rand", "encoding/gob", "encoding/json", "expvar", "go/token", "log", "math/big", "math/rand", "regexp", "time": - for _, spec := range file.Imports { - path, _ := strconv.Unquote(spec.Path.Value) - if path == "sync" { - if spec.Name == nil { - spec.Name = ast.NewIdent("sync") + files = append(files, file) + } + + if errList != nil { + return nil, errList + } + return files, nil +} + +// augmentOverlayFile is the part of parseAndAugment that processes +// an overlay file AST to collect information such as compiler directives and +// perform any initial augmentation needed to the overlay. +func augmentOverlayFile(file *ast.File, replacedDeclNames map[string]overrideInfo) { + for i, decl := range file.Decls { + purgeDecl := astutil.Purge(decl) + switch d := decl.(type) { + case *ast.FuncDecl: + k := astutil.FuncKey(d) + replacedDeclNames[k] = overrideInfo{ + keepOriginal: astutil.KeepOriginal(d), + pruneFuncBody: astutil.PruneOriginal(d), + } + case *ast.GenDecl: + for j, spec := range d.Specs { + purgeSpec := purgeDecl || astutil.Purge(spec) + switch d.Tok { + case token.TYPE: + replacedDeclNames[spec.(*ast.TypeSpec).Name.Name] = overrideInfo{ + purgeMethods: purgeSpec, } - spec.Path.Value = `"github.com/gopherjs/gopherjs/nosync"` + case token.VAR, token.CONST: + for _, name := range spec.(*ast.ValueSpec).Names { + replacedDeclNames[name.Name] = overrideInfo{} + } + } + if purgeSpec { + d.Specs[j] = nil } } + d.Specs = astutil.Squeeze(d.Specs) + if len(d.Specs) == 0 { + file.Decls[i] = nil + } + } + if purgeDecl { + file.Decls[i] = nil } + } + file.Decls = astutil.Squeeze(file.Decls) +} - for _, decl := range file.Decls { - switch d := decl.(type) { - case *ast.FuncDecl: - k := astutil.FuncKey(d) - if info, ok := replacedDeclNames[k]; ok { - if info.pruneOriginal { - // Prune function bodies, since it may contain code invalid for - // GopherJS and pin unwanted imports. - d.Body = nil - } - if info.keepOriginal { - // Allow overridden function calls - // The standard library implementation of foo() becomes _gopherjs_original_foo() - d.Name.Name = "_gopherjs_original_" + d.Name.Name - } else { - d.Name = ast.NewIdent("_") - } +// augmentOriginalImports is the part of parseAndAugment that processes +// an original file AST to modify the imports for that file. +func augmentOriginalImports(importPath string, file *ast.File) { + switch importPath { + case "crypto/rand", "encoding/gob", "encoding/json", "expvar", "go/token", "log", "math/big", "math/rand", "regexp", "time": + for _, spec := range file.Imports { + path, _ := strconv.Unquote(spec.Path.Value) + if path == "sync" { + if spec.Name == nil { + spec.Name = ast.NewIdent("sync") } - case *ast.GenDecl: + spec.Path.Value = `"github.com/gopherjs/gopherjs/nosync"` + } + } + } +} + +// augmentOriginalFile is the part of parseAndAugment that processes +// an original file AST to augment the source code using the directives +// from the overlay. +func augmentOriginalFile(file *ast.File, replacedDeclNames map[string]overrideInfo) { + for i, decl := range file.Decls { + switch d := decl.(type) { + case *ast.FuncDecl: + k := astutil.FuncKey(d) + if info, ok := replacedDeclNames[k]; ok { + if info.pruneFuncBody { + // Prune function bodies, since it may contain code invalid for + // GopherJS and pin unwanted imports. + d.Body = nil + } + if info.keepOriginal { + // Allow overridden function calls + // The standard library implementation of foo() becomes _gopherjs_original_foo() + d.Name.Name = "_gopherjs_original_" + d.Name.Name + } else { + file.Decls[i] = nil + } + } else if recvKey := astutil.FuncReceiverKey(d); len(recvKey) > 0 { + // check if the receiver has been purged, if so, remove the method too. + if info, ok := replacedDeclNames[recvKey]; ok && info.purgeMethods { + file.Decls[i] = nil + } + } + case *ast.GenDecl: + for j, spec := range d.Specs { switch d.Tok { case token.TYPE: - for _, spec := range d.Specs { - s := spec.(*ast.TypeSpec) - if _, ok := replacedDeclNames[s.Name.Name]; ok { - s.Name = ast.NewIdent("_") - s.Type = &ast.StructType{Struct: s.Pos(), Fields: &ast.FieldList{}} - s.TypeParams = nil - } + if _, ok := replacedDeclNames[spec.(*ast.TypeSpec).Name.Name]; ok { + d.Specs[j] = nil } case token.VAR, token.CONST: - for _, spec := range d.Specs { - s := spec.(*ast.ValueSpec) - for i, name := range s.Names { - if _, ok := replacedDeclNames[name.Name]; ok { - s.Names[i] = ast.NewIdent("_") - } + s := spec.(*ast.ValueSpec) + for k, name := range s.Names { + if _, ok := replacedDeclNames[name.Name]; ok { + s.Names[k] = nil } } + s.Names = astutil.Squeeze(s.Names) + if len(s.Names) == 0 { + d.Specs[j] = nil + } } } + d.Specs = astutil.Squeeze(d.Specs) + if len(d.Specs) == 0 { + file.Decls[i] = nil + } } + } + file.Decls = astutil.Squeeze(file.Decls) +} - files = append(files, file) +// pruneImports will remove any unused imports from the file. +// +// This will not remove any unnamed (`.`) or unused (`_`) imports. +func pruneImports(file *ast.File) { + unused := make(map[string]int, len(file.Imports)) + for i, in := range file.Imports { + if name := astutil.ImportName(in); len(name) > 0 { + unused[name] = i + } } - if errList != nil { - return nil, nil, errList + ast.Walk(astutil.NewCallbackVisitor(func(n ast.Node) bool { + if sel, ok := n.(*ast.SelectorExpr); ok { + if id, ok := sel.X.(*ast.Ident); ok && id.Obj == nil { + delete(unused, id.Name) + } + } + return len(unused) > 0 + }), file) + + for _, i := range unused { + file.Imports[i] = nil } - return files, jsFiles, nil + + file.Imports = astutil.Squeeze(file.Imports) } // Options controls build process behavior. @@ -678,7 +789,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { archive := s.buildCache.LoadArchive(pkg.ImportPath) if archive != nil && !pkg.SrcModTime.After(archive.BuildTime) { if err := archive.RegisterTypes(s.Types); err != nil { - panic(fmt.Errorf("Failed to load type information from %v: %w", archive, err)) + panic(fmt.Errorf("failed to load type information from %v: %w", archive, err)) } s.UpToDateArchives[pkg.ImportPath] = archive // Existing archive is up to date, no need to build it from scratch. diff --git a/compiler/astutil/astutil.go b/compiler/astutil/astutil.go index e14bcf267..9bde946af 100644 --- a/compiler/astutil/astutil.go +++ b/compiler/astutil/astutil.go @@ -5,7 +5,10 @@ import ( "go/ast" "go/token" "go/types" + "path" + "reflect" "regexp" + "strconv" ) func RemoveParens(e ast.Expr) ast.Expr { @@ -59,17 +62,72 @@ func ImportsUnsafe(file *ast.File) bool { return false } +// ImportName tries to determine the package name for an import. +// +// If the package name isn't specified then this will make a best +// make a best guess using the import path. +// If the import name is unnamed (`.`), unused (`_`), or there +// was an issue determining the package name then empty is returned. +func ImportName(spec *ast.ImportSpec) string { + if spec == nil { + return `` + } + + var name string + if spec.Name != nil { + name = spec.Name.Name + } else { + // Package name is not specified so try to guess + // it based on the import path. + importPath, err := strconv.Unquote(spec.Path.Value) + if err != nil { + return `` + } + name = path.Base(importPath) + } + + switch name { + case `_`, `.`, `/`: + return `` + default: + return name + } +} + // FuncKey returns a string, which uniquely identifies a top-level function or // method in a package. func FuncKey(d *ast.FuncDecl) string { - if d.Recv == nil || len(d.Recv.List) == 0 { - return d.Name.Name + if recvKey := FuncReceiverKey(d); len(recvKey) > 0 { + return recvKey + "." + d.Name.Name + } + return d.Name.Name +} + +// FuncReceiverKey returns a string that uniquely identifies the receiver +// struct of the function or an empty string if there is no receiver. +// This name will match the name of the struct in the struct's type spec. +func FuncReceiverKey(d *ast.FuncDecl) string { + if d == nil || d.Recv == nil || len(d.Recv.List) == 0 { + return `` } recv := d.Recv.List[0].Type - if star, ok := recv.(*ast.StarExpr); ok { - recv = star.X + for { + switch r := recv.(type) { + case *ast.IndexListExpr: + recv = r.X + continue + case *ast.IndexExpr: + recv = r.X + continue + case *ast.StarExpr: + recv = r.X + continue + case *ast.Ident: + return r.Name + default: + panic(fmt.Errorf(`unexpected type %T in receiver of function: %v`, recv, d)) + } } - return recv.(*ast.Ident).Name + "." + d.Name.Name } // anyDocLine calls the given predicate on all associated documentation @@ -109,24 +167,20 @@ func anyDocLine(node any, predicate func(line string) bool) bool { // directiveMatcher is a regex which matches a GopherJS directive // and finds the directive action. -// -// This matches the largest directive action until whitespace or EOL -// to differentiate from any directive action which is a prefix -// for another directive action. var directiveMatcher = regexp.MustCompile(`^\/(?:\/|\*)gopherjs:([\w-]+)`) // hasDirective returns true if the associated documentation // or line comments for the given node have the given directive action. // -// All gopherjs directives must start with `//gopherjs:` or `/*gopherjs:` -// and followed by an action without any whitespace. The action must be -// one or more letter, decimal, underscore, or hyphen. +// All GopherJS-specific directives must start with `//gopherjs:` or +// `/*gopherjs:` and followed by an action without any whitespace. The action +// must be one or more letter, decimal, underscore, or hyphen. // // see https://pkg.go.dev/cmd/compile#hdr-Compiler_Directives -func hasDirective(node any, directive string) bool { +func hasDirective(node any, directiveAction string) bool { return anyDocLine(node, func(line string) bool { m := directiveMatcher.FindStringSubmatch(line) - return len(m) == 2 && m[1] == directive + return len(m) == 2 && m[1] == directiveAction }) } @@ -151,20 +205,21 @@ func PruneOriginal(d *ast.FuncDecl) bool { // logic to expose the original function such that it can be called. For a // function in the original called `foo`, it will be accessible by the name // `_gopherjs_original_foo`. -func KeepOriginal(d *ast.FuncDecl) bool { +func KeepOriginal(d any) bool { return hasDirective(d, `keep-original`) } // Purge returns true if gopherjs:purge directive is present -// on a struct, interface, variables, constants, and functions. +// on a struct, interface, variable, constant, or function. // // `//gopherjs:purge` is a GopherJS-specific directive, which can be // applied in native overlays and will instruct the augmentation logic to // delete part of the standard library without a replacement. This directive // can be used to remove code that would be invalid in GopherJS, such as code -// using unsupported features (e.g. generic interfaces and methods before -// generics were fully supported). It should be used with caution since it -// may remove needed dependencies. +// using unsupported features (e.g. generic interfaces before generics were +// fully supported). It should be used with caution since it may remove needed +// dependencies. If a struct is purged, all methods using that struct as +// a receiver will also be purged. func Purge(d any) bool { return hasDirective(d, `purge`) } @@ -223,3 +278,37 @@ func EndsWithReturn(stmts []ast.Stmt) bool { return false } } + +// Squeeze removes all nil nodes from the slice. +// +// The given slice will be modified. This is designed for squeezing +// declaration, specification, imports, and identifier lists. +func Squeeze[E ast.Node, S ~[]E](s S) S { + var zero E + count, dest := len(s), 0 + for src := 0; src < count; src++ { + if !reflect.DeepEqual(s[src], zero) { + // Swap the values, this will put the nil values to the end + // of the slice so that the tail isn't holding onto pointers. + s[dest], s[src] = s[src], s[dest] + dest++ + } + } + return s[:dest] +} + +type CallbackVisitor struct { + predicate func(node ast.Node) bool +} + +func NewCallbackVisitor(predicate func(node ast.Node) bool) *CallbackVisitor { + return &CallbackVisitor{predicate: predicate} +} + +func (v *CallbackVisitor) Visit(node ast.Node) ast.Visitor { + if v.predicate == nil || node == nil || !v.predicate(node) { + v.predicate = nil + return nil + } + return v +} diff --git a/compiler/astutil/astutil_test.go b/compiler/astutil/astutil_test.go index 622b9f3ee..6b80431e3 100644 --- a/compiler/astutil/astutil_test.go +++ b/compiler/astutil/astutil_test.go @@ -4,6 +4,7 @@ import ( "fmt" "go/ast" "go/token" + "strconv" "testing" "github.com/gopherjs/gopherjs/internal/srctesting" @@ -61,24 +62,41 @@ func TestFuncKey(t *testing.T) { want string }{ { - desc: "top-level function", - src: `package testpackage; func foo() {}`, - want: "foo", + desc: `top-level function`, + src: `func foo() {}`, + want: `foo`, + }, { + desc: `top-level exported function`, + src: `func Foo() {}`, + want: `Foo`, + }, { + desc: `method on reference`, + src: `func (_ myType) bar() {}`, + want: `myType.bar`, + }, { + desc: `method on pointer`, + src: ` func (_ *myType) bar() {}`, + want: `myType.bar`, }, { - desc: "top-level exported function", - src: `package testpackage; func Foo() {}`, - want: "Foo", + desc: `method on generic reference`, + src: ` func (_ myType[T]) bar() {}`, + want: `myType.bar`, }, { - desc: "method", - src: `package testpackage; func (_ myType) bar() {}`, - want: "myType.bar", + desc: `method on generic pointer`, + src: ` func (_ *myType[T]) bar() {}`, + want: `myType.bar`, + }, { + desc: `method on struct with multiple generics`, + src: ` func (_ *myType[T1, T2, T3, T4]) bar() {}`, + want: `myType.bar`, }, } for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - fdecl := srctesting.ParseFuncDecl(t, test.src) + src := `package testpackage; ` + test.src + fdecl := srctesting.ParseFuncDecl(t, src) if got := FuncKey(fdecl); got != test.want { - t.Errorf("Got %q, want %q", got, test.want) + t.Errorf(`Got %q, want %q`, got, test.want) } }) } @@ -545,3 +563,61 @@ func TestEndsWithReturn(t *testing.T) { }) } } + +func TestSqueezeIdents(t *testing.T) { + tests := []struct { + desc string + count int + assign []int + }{ + { + desc: `no squeezing`, + count: 5, + assign: []int{0, 1, 2, 3, 4}, + }, { + desc: `missing front`, + count: 5, + assign: []int{3, 4}, + }, { + desc: `missing back`, + count: 5, + assign: []int{0, 1, 2}, + }, { + desc: `missing several`, + count: 10, + assign: []int{1, 2, 3, 6, 8}, + }, { + desc: `empty`, + count: 0, + assign: []int{}, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + input := make([]*ast.Ident, test.count) + for _, i := range test.assign { + input[i] = ast.NewIdent(strconv.Itoa(i)) + } + + result := Squeeze(input) + if len(result) != len(test.assign) { + t.Errorf("Squeeze() returned a slice %d long, want %d", len(result), len(test.assign)) + } + for i, id := range input { + if i < len(result) { + if id == nil { + t.Errorf(`Squeeze() returned a nil in result at %d`, i) + } else { + value, err := strconv.Atoi(id.Name) + if err != nil || value != test.assign[i] { + t.Errorf(`Squeeze() returned %s at %d instead of %d`, id.Name, i, test.assign[i]) + } + } + } else if id != nil { + t.Errorf(`Squeeze() didn't clear out tail of slice, want %d nil`, i) + } + } + }) + } +} diff --git a/compiler/natives/src/net/http/http.go b/compiler/natives/src/net/http/http.go index 7843235b2..8fd607c4d 100644 --- a/compiler/natives/src/net/http/http.go +++ b/compiler/natives/src/net/http/http.go @@ -7,7 +7,7 @@ import ( "bufio" "bytes" "errors" - "io/ioutil" + "io" "net/textproto" "strconv" @@ -68,7 +68,7 @@ func (t *XHRTransport) RoundTrip(req *Request) (*Response, error) { StatusCode: xhr.Get("status").Int(), Header: Header(header), ContentLength: contentLength, - Body: ioutil.NopCloser(bytes.NewReader(body)), + Body: io.NopCloser(bytes.NewReader(body)), Request: req, } }) @@ -91,7 +91,7 @@ func (t *XHRTransport) RoundTrip(req *Request) (*Response, error) { if req.Body == nil { xhr.Call("send") } else { - body, err := ioutil.ReadAll(req.Body) + body, err := io.ReadAll(req.Body) if err != nil { req.Body.Close() // RoundTrip must always close the body, including on errors. return nil, err From aa4f22330adad41134c530aef25513d802c322cb Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Wed, 13 Dec 2023 14:15:29 -0700 Subject: [PATCH 08/11] testing build updates --- build/build.go | 166 ++++++++++++++++++++----------- build/build_test.go | 112 +++++++++++++++++++++ compiler/astutil/astutil.go | 23 +---- compiler/astutil/astutil_test.go | 54 ++++++++-- 4 files changed, 270 insertions(+), 85 deletions(-) diff --git a/build/build.go b/build/build.go index 6e5c450ab..6cd739c5a 100644 --- a/build/build.go +++ b/build/build.go @@ -124,12 +124,10 @@ type overrideInfo struct { // but the identifier will be prefixed by `_gopherjs_original_foo`. keepOriginal bool - // pruneFuncBody indicates that the body of the function should - // be removed to prevent statements that are invalid in GopherJS. - pruneFuncBody bool - // purgeMethods indicates that this info is for a struct and // if a method has this struct as a receiver should also be removed. + // If the method is defined in the overlays and therefore has its + // own overrides, this will be ignored. purgeMethods bool } @@ -146,11 +144,9 @@ type overrideInfo struct { // - For function identifiers that exist in the original and the overrides and have the // directive `gopherjs:keep-original`, the original identifier in the AST gets // prefixed by `_gopherjs_original_`. -// - For functions that exist in the original and the overrides and have the -// directive `gopherjs:prune-original`, the original function body in the AST -// is removed. This removes code that is invalid in GopherJS. // - For identifiers that exist in the original and the overrides // and have the directive `gopherjs:purge`, both the original and override are removed. +// This is for completely removing something which is currently invalid for GopherJS. // - Otherwise for identifiers that exist in the original and the overrides, // the original identifier is removed. // - New identifiers that don't exist in original package get added. @@ -162,17 +158,17 @@ func parseAndAugment(xctx XContext, pkg *PackageData, isTest bool, fileSet *toke return nil, nil, err } - replacedDeclNames := make(map[string]overrideInfo) + overrides := make(map[string]overrideInfo) for _, file := range overlayFiles { - augmentOverlayFile(file, replacedDeclNames) - pruneImports(file) + augmentOverlayFile(fileSet, file, overrides) + pruneImports(fileSet, file) } - delete(replacedDeclNames, "init") + delete(overrides, "init") for _, file := range originalFiles { augmentOriginalImports(pkg.ImportPath, file) - augmentOriginalFile(file, replacedDeclNames) - pruneImports(file) + augmentOriginalFile(fileSet, file, overrides) + pruneImports(fileSet, file) } return append(overlayFiles, originalFiles...), jsFiles, nil @@ -262,43 +258,53 @@ func parserOriginalFiles(pkg *PackageData, fileSet *token.FileSet) ([]*ast.File, // augmentOverlayFile is the part of parseAndAugment that processes // an overlay file AST to collect information such as compiler directives and // perform any initial augmentation needed to the overlay. -func augmentOverlayFile(file *ast.File, replacedDeclNames map[string]overrideInfo) { +func augmentOverlayFile(fileSet *token.FileSet, file *ast.File, overrides map[string]overrideInfo) { + commentMap := ast.NewCommentMap(fileSet, file, file.Comments) + fileChanged := false for i, decl := range file.Decls { purgeDecl := astutil.Purge(decl) switch d := decl.(type) { case *ast.FuncDecl: k := astutil.FuncKey(d) - replacedDeclNames[k] = overrideInfo{ - keepOriginal: astutil.KeepOriginal(d), - pruneFuncBody: astutil.PruneOriginal(d), + overrides[k] = overrideInfo{ + keepOriginal: astutil.KeepOriginal(d), } case *ast.GenDecl: + declChange := false for j, spec := range d.Specs { purgeSpec := purgeDecl || astutil.Purge(spec) - switch d.Tok { - case token.TYPE: - replacedDeclNames[spec.(*ast.TypeSpec).Name.Name] = overrideInfo{ + switch s := spec.(type) { + case *ast.TypeSpec: + overrides[s.Name.Name] = overrideInfo{ purgeMethods: purgeSpec, } - case token.VAR, token.CONST: - for _, name := range spec.(*ast.ValueSpec).Names { - replacedDeclNames[name.Name] = overrideInfo{} + case *ast.ValueSpec: + for _, name := range s.Names { + overrides[name.Name] = overrideInfo{} } } if purgeSpec { + declChange = true d.Specs[j] = nil } } - d.Specs = astutil.Squeeze(d.Specs) - if len(d.Specs) == 0 { - file.Decls[i] = nil + if declChange { + d.Specs = astutil.Squeeze(d.Specs) + if len(d.Specs) == 0 { + fileChanged = true + file.Decls[i] = nil + } } } if purgeDecl { + fileChanged = true file.Decls[i] = nil } } - file.Decls = astutil.Squeeze(file.Decls) + if fileChanged { + file.Decls = astutil.Squeeze(file.Decls) + file.Comments = commentMap.Filter(file).Comments() + } } // augmentOriginalImports is the part of parseAndAugment that processes @@ -318,66 +324,77 @@ func augmentOriginalImports(importPath string, file *ast.File) { } } -// augmentOriginalFile is the part of parseAndAugment that processes -// an original file AST to augment the source code using the directives -// from the overlay. -func augmentOriginalFile(file *ast.File, replacedDeclNames map[string]overrideInfo) { +// augmentOriginalFile is the part of parseAndAugment that processes an +// original file AST to augment the source code using the overrides from +// the overlay files. +func augmentOriginalFile(fileSet *token.FileSet, file *ast.File, overrides map[string]overrideInfo) { + commentMap := ast.NewCommentMap(fileSet, file, file.Comments) + fileChanged := false for i, decl := range file.Decls { switch d := decl.(type) { case *ast.FuncDecl: k := astutil.FuncKey(d) - if info, ok := replacedDeclNames[k]; ok { - if info.pruneFuncBody { - // Prune function bodies, since it may contain code invalid for - // GopherJS and pin unwanted imports. - d.Body = nil - } + if info, ok := overrides[k]; ok { if info.keepOriginal { // Allow overridden function calls // The standard library implementation of foo() becomes _gopherjs_original_foo() - d.Name.Name = "_gopherjs_original_" + d.Name.Name + d.Name.Name = `_gopherjs_original_` + d.Name.Name } else { + fileChanged = true file.Decls[i] = nil } } else if recvKey := astutil.FuncReceiverKey(d); len(recvKey) > 0 { // check if the receiver has been purged, if so, remove the method too. - if info, ok := replacedDeclNames[recvKey]; ok && info.purgeMethods { + if info, ok := overrides[recvKey]; ok && info.purgeMethods { + fileChanged = true file.Decls[i] = nil } } case *ast.GenDecl: + declChanged := false for j, spec := range d.Specs { - switch d.Tok { - case token.TYPE: - if _, ok := replacedDeclNames[spec.(*ast.TypeSpec).Name.Name]; ok { + switch s := spec.(type) { + case *ast.TypeSpec: + if _, ok := overrides[s.Name.Name]; ok { + declChanged = true d.Specs[j] = nil } - case token.VAR, token.CONST: - s := spec.(*ast.ValueSpec) + case *ast.ValueSpec: + specChanged := false for k, name := range s.Names { - if _, ok := replacedDeclNames[name.Name]; ok { + if _, ok := overrides[name.Name]; ok { + specChanged = true s.Names[k] = nil } } - s.Names = astutil.Squeeze(s.Names) - if len(s.Names) == 0 { - d.Specs[j] = nil + if specChanged { + s.Names = astutil.Squeeze(s.Names) + if len(s.Names) == 0 { + declChanged = true + d.Specs[j] = nil + } } } } - d.Specs = astutil.Squeeze(d.Specs) - if len(d.Specs) == 0 { - file.Decls[i] = nil + if declChanged { + d.Specs = astutil.Squeeze(d.Specs) + if len(d.Specs) == 0 { + fileChanged = true + file.Decls[i] = nil + } } } } - file.Decls = astutil.Squeeze(file.Decls) + if fileChanged { + file.Decls = astutil.Squeeze(file.Decls) + file.Comments = commentMap.Filter(file).Comments() + } } // pruneImports will remove any unused imports from the file. // -// This will not remove any unnamed (`.`) or unused (`_`) imports. -func pruneImports(file *ast.File) { +// This will not remove any dot (`.`) or blank (`_`) imports. +func pruneImports(fileSet *token.FileSet, file *ast.File) { unused := make(map[string]int, len(file.Imports)) for i, in := range file.Imports { if name := astutil.ImportName(in); len(name) > 0 { @@ -385,6 +402,7 @@ func pruneImports(file *ast.File) { } } + // Remove any import which is used in a selector. ast.Walk(astutil.NewCallbackVisitor(func(n ast.Node) bool { if sel, ok := n.(*ast.SelectorExpr); ok { if id, ok := sel.X.(*ast.Ident); ok && id.Obj == nil { @@ -394,11 +412,43 @@ func pruneImports(file *ast.File) { return len(unused) > 0 }), file) - for _, i := range unused { - file.Imports[i] = nil - } + if len(unused) > 0 { + commentMap := ast.NewCommentMap(fileSet, file, file.Comments) + + // Remove all import specifications + isUnusedSpec := map[*ast.ImportSpec]bool{} + for _, index := range unused { + isUnusedSpec[file.Imports[index]] = true + } + for i, decl := range file.Decls { + if d, ok := decl.(*ast.GenDecl); ok { + declChanged := false + for j, spec := range d.Specs { + if other, ok := spec.(*ast.ImportSpec); ok { + if isUnusedSpec[other] { + declChanged = true + d.Specs[j] = nil + } + } + } + if declChanged { + d.Specs = astutil.Squeeze(d.Specs) + if len(d.Specs) == 0 { + file.Decls[i] = nil + } + } + } + } - file.Imports = astutil.Squeeze(file.Imports) + // Remove the import copies in the file + for _, index := range unused { + file.Imports[index] = nil + } + + file.Decls = astutil.Squeeze(file.Decls) + file.Imports = astutil.Squeeze(file.Imports) + file.Comments = commentMap.Filter(file).Comments() + } } // Options controls build process behavior. diff --git a/build/build_test.go b/build/build_test.go index 2fa17e2c5..a37280e6d 100644 --- a/build/build_test.go +++ b/build/build_test.go @@ -1,12 +1,15 @@ package build import ( + "bytes" "fmt" gobuild "go/build" + "go/printer" "go/token" "strconv" "testing" + "github.com/gopherjs/gopherjs/internal/srctesting" "github.com/shurcooL/go/importgraphutil" ) @@ -127,3 +130,112 @@ func (m stringSet) String() string { } return fmt.Sprintf("%q", s) } + +func TestOverlayAugmentation(t *testing.T) { + tests := []struct { + desc string + src string + want string + expInfo map[string]overrideInfo + }{ + { + desc: `prune an unused import`, + src: `import foo "some/other/bar"`, + want: ``, + expInfo: map[string]overrideInfo{}, + }, { + desc: `remove function`, + src: `func Foo(a, b int) int { + return a + b + }`, + want: `func Foo(a, b int) int { + return a + b + }`, + expInfo: map[string]overrideInfo{ + `Foo`: {}, + }, + }, { + desc: `keep function`, + src: `//gopherjs:keep-original + func Foo(a, b int) int { + return a + b + }`, + want: `//gopherjs:keep-original + func Foo(a, b int) int { + return a + b + }`, + expInfo: map[string]overrideInfo{ + `Foo`: {keepOriginal: true}, + }, + }, { + desc: `purge function`, + src: `//gopherjs:purge + func Foo(a, b int) int { + return a + b + }`, + want: ``, + expInfo: map[string]overrideInfo{ + `Foo`: {}, + }, + }, { + desc: `purge struct removes an import`, + src: `import "bytes" + import "math" + + //gopherjs:purge + type Foo struct { + bar *bytes.Buffer + } + + const Tau = math.Pi * 2.0`, + want: `import "math" + + const Tau = math.Pi * 2.0`, + expInfo: map[string]overrideInfo{ + `Foo`: {purgeMethods: true}, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + pkgName := "package testpackage\n\n" + fsetSrc := token.NewFileSet() + fileSrc := srctesting.Parse(t, fsetSrc, pkgName+test.src) + + overrides := map[string]overrideInfo{} + augmentOverlayFile(fsetSrc, fileSrc, overrides) + pruneImports(fsetSrc, fileSrc) + + buf := &bytes.Buffer{} + _ = printer.Fprint(buf, fsetSrc, fileSrc) + got := buf.String() + + fsetWant := token.NewFileSet() + fileWant := srctesting.Parse(t, fsetWant, pkgName+test.want) + + buf.Reset() + _ = printer.Fprint(buf, fsetWant, fileWant) + want := buf.String() + + if got != want { + t.Errorf("augmentOverlayFile and pruneImports got unexpected code:\n"+ + "returned:\n\t%q\nwant:\n\t%q", got, want) + } + + for key, expInfo := range test.expInfo { + if gotInfo, ok := overrides[key]; !ok { + t.Errorf(`%q was expected but not gotten`, key) + } else if expInfo != gotInfo { + t.Errorf(`%q had wrong info, got %+v`, key, gotInfo) + } + } + for key, gotInfo := range overrides { + if _, ok := test.expInfo[key]; !ok { + t.Errorf(`%q with %+v was not expected`, key, gotInfo) + } + } + }) + } + +} diff --git a/compiler/astutil/astutil.go b/compiler/astutil/astutil.go index 9bde946af..9f4bae215 100644 --- a/compiler/astutil/astutil.go +++ b/compiler/astutil/astutil.go @@ -53,6 +53,7 @@ func IsTypeExpr(expr ast.Expr, info *types.Info) bool { } } +// ImportsUnsafe determines if this file imports the "unsafe" package. func ImportsUnsafe(file *ast.File) bool { for _, imp := range file.Imports { if imp.Path.Value == `"unsafe"` { @@ -66,7 +67,7 @@ func ImportsUnsafe(file *ast.File) bool { // // If the package name isn't specified then this will make a best // make a best guess using the import path. -// If the import name is unnamed (`.`), unused (`_`), or there +// If the import name is dot (`.`), blank (`_`), or there // was an issue determining the package name then empty is returned. func ImportName(spec *ast.ImportSpec) string { if spec == nil { @@ -77,12 +78,7 @@ func ImportName(spec *ast.ImportSpec) string { if spec.Name != nil { name = spec.Name.Name } else { - // Package name is not specified so try to guess - // it based on the import path. - importPath, err := strconv.Unquote(spec.Path.Value) - if err != nil { - return `` - } + importPath, _ := strconv.Unquote(spec.Path.Value) name = path.Base(importPath) } @@ -184,19 +180,6 @@ func hasDirective(node any, directiveAction string) bool { }) } -// PruneOriginal returns true if gopherjs:prune-original directive is present -// before a function decl. -// -// `//gopherjs:prune-original` is a GopherJS-specific directive, which can be -// applied to functions in native overlays and will instruct the augmentation -// logic to delete the body of a standard library function that was replaced. -// This directive can be used to remove code that would be invalid in GopherJS, -// such as code expecting ints to be 64-bit. It should be used with caution -// since it may create unused imports in the original source file. -func PruneOriginal(d *ast.FuncDecl) bool { - return hasDirective(d, `prune-original`) -} - // KeepOriginal returns true if gopherjs:keep-original directive is present // before a function decl. // diff --git a/compiler/astutil/astutil_test.go b/compiler/astutil/astutil_test.go index 6b80431e3..20a9c2d1c 100644 --- a/compiler/astutil/astutil_test.go +++ b/compiler/astutil/astutil_test.go @@ -55,6 +55,47 @@ func TestImportsUnsafe(t *testing.T) { } } +func TestImportName(t *testing.T) { + tests := []struct { + desc string + src string + want string + }{ + { + desc: `named import`, + src: `import foo "some/other/bar"`, + want: `foo`, + }, { + desc: `unnamed import`, + src: `import "some/other/bar"`, + want: `bar`, + }, { + desc: `dot import`, + src: `import . "some/other/bar"`, + want: ``, + }, { + desc: `blank import`, + src: `import _ "some/other/bar"`, + want: ``, + }, + } + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + src := "package testpackage\n\n" + test.src + fset := token.NewFileSet() + file := srctesting.Parse(t, fset, src) + if len(file.Imports) != 1 { + t.Fatal(`expected one and only one import`) + } + importSpec := file.Imports[0] + got := ImportName(importSpec) + if got != test.want { + t.Fatalf(`ImportName() returned %q, want %q`, got, test.want) + } + }) + } +} + func TestFuncKey(t *testing.T) { tests := []struct { desc string @@ -102,7 +143,7 @@ func TestFuncKey(t *testing.T) { } } -func TestPruneOriginal(t *testing.T) { +func TestKeepOriginal(t *testing.T) { tests := []struct { desc string src string @@ -122,20 +163,20 @@ func TestPruneOriginal(t *testing.T) { }, { desc: "only directive", src: `package testpackage; - //gopherjs:prune-original + //gopherjs:keep-original func foo() {}`, want: true, }, { desc: "directive with explanation", src: `package testpackage; - //gopherjs:prune-original because reasons + //gopherjs:keep-original because reasons func foo() {}`, want: true, }, { desc: "directive in godoc", src: `package testpackage; // foo does something - //gopherjs:prune-original + //gopherjs:keep-original func foo() {}`, want: true, }, @@ -143,8 +184,8 @@ func TestPruneOriginal(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { fdecl := srctesting.ParseFuncDecl(t, test.src) - if got := PruneOriginal(fdecl); got != test.want { - t.Errorf("PruneOriginal() returned %t, want %t", got, test.want) + if got := KeepOriginal(fdecl); got != test.want { + t.Errorf("KeepOriginal() returned %t, want %t", got, test.want) } }) } @@ -499,7 +540,6 @@ func TestHasDirectiveBadCase(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { const action = `do-stuff` - var got string func() { defer func() { got = fmt.Sprint(recover()) }() From f828f20ac6f722541fbbbb140076c09b49d5f967 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Wed, 13 Dec 2023 16:08:24 -0700 Subject: [PATCH 09/11] testing build updates --- build/build.go | 139 ++++++++++++++++++------------------ build/build_test.go | 44 ++++++++++++ compiler/astutil/astutil.go | 12 ++-- 3 files changed, 118 insertions(+), 77 deletions(-) diff --git a/build/build.go b/build/build.go index 6cd739c5a..dcb531b7c 100644 --- a/build/build.go +++ b/build/build.go @@ -124,8 +124,8 @@ type overrideInfo struct { // but the identifier will be prefixed by `_gopherjs_original_foo`. keepOriginal bool - // purgeMethods indicates that this info is for a struct and - // if a method has this struct as a receiver should also be removed. + // purgeMethods indicates that this info is for a type and + // if a method has this type as a receiver should also be removed. // If the method is defined in the overlays and therefore has its // own overrides, this will be ignored. purgeMethods bool @@ -147,6 +147,7 @@ type overrideInfo struct { // - For identifiers that exist in the original and the overrides // and have the directive `gopherjs:purge`, both the original and override are removed. // This is for completely removing something which is currently invalid for GopherJS. +// For any purged types any methods with that type as the receiver are also removed. // - Otherwise for identifiers that exist in the original and the overrides, // the original identifier is removed. // - New identifiers that don't exist in original package get added. @@ -259,8 +260,6 @@ func parserOriginalFiles(pkg *PackageData, fileSet *token.FileSet) ([]*ast.File, // an overlay file AST to collect information such as compiler directives and // perform any initial augmentation needed to the overlay. func augmentOverlayFile(fileSet *token.FileSet, file *ast.File, overrides map[string]overrideInfo) { - commentMap := ast.NewCommentMap(fileSet, file, file.Comments) - fileChanged := false for i, decl := range file.Decls { purgeDecl := astutil.Purge(decl) switch d := decl.(type) { @@ -270,7 +269,6 @@ func augmentOverlayFile(fileSet *token.FileSet, file *ast.File, overrides map[st keepOriginal: astutil.KeepOriginal(d), } case *ast.GenDecl: - declChange := false for j, spec := range d.Specs { purgeSpec := purgeDecl || astutil.Purge(spec) switch s := spec.(type) { @@ -284,27 +282,15 @@ func augmentOverlayFile(fileSet *token.FileSet, file *ast.File, overrides map[st } } if purgeSpec { - declChange = true d.Specs[j] = nil } } - if declChange { - d.Specs = astutil.Squeeze(d.Specs) - if len(d.Specs) == 0 { - fileChanged = true - file.Decls[i] = nil - } - } } if purgeDecl { - fileChanged = true file.Decls[i] = nil } } - if fileChanged { - file.Decls = astutil.Squeeze(file.Decls) - file.Comments = commentMap.Filter(file).Comments() - } + finalizeRemovals(file) } // augmentOriginalImports is the part of parseAndAugment that processes @@ -328,67 +314,41 @@ func augmentOriginalImports(importPath string, file *ast.File) { // original file AST to augment the source code using the overrides from // the overlay files. func augmentOriginalFile(fileSet *token.FileSet, file *ast.File, overrides map[string]overrideInfo) { - commentMap := ast.NewCommentMap(fileSet, file, file.Comments) - fileChanged := false for i, decl := range file.Decls { switch d := decl.(type) { case *ast.FuncDecl: - k := astutil.FuncKey(d) - if info, ok := overrides[k]; ok { + if info, ok := overrides[astutil.FuncKey(d)]; ok { if info.keepOriginal { // Allow overridden function calls // The standard library implementation of foo() becomes _gopherjs_original_foo() d.Name.Name = `_gopherjs_original_` + d.Name.Name } else { - fileChanged = true file.Decls[i] = nil } } else if recvKey := astutil.FuncReceiverKey(d); len(recvKey) > 0 { // check if the receiver has been purged, if so, remove the method too. if info, ok := overrides[recvKey]; ok && info.purgeMethods { - fileChanged = true file.Decls[i] = nil } } case *ast.GenDecl: - declChanged := false for j, spec := range d.Specs { switch s := spec.(type) { case *ast.TypeSpec: if _, ok := overrides[s.Name.Name]; ok { - declChanged = true d.Specs[j] = nil } case *ast.ValueSpec: - specChanged := false for k, name := range s.Names { if _, ok := overrides[name.Name]; ok { - specChanged = true s.Names[k] = nil } } - if specChanged { - s.Names = astutil.Squeeze(s.Names) - if len(s.Names) == 0 { - declChanged = true - d.Specs[j] = nil - } - } - } - } - if declChanged { - d.Specs = astutil.Squeeze(d.Specs) - if len(d.Specs) == 0 { - fileChanged = true - file.Decls[i] = nil } } } } - if fileChanged { - file.Decls = astutil.Squeeze(file.Decls) - file.Comments = commentMap.Filter(file).Comments() - } + finalizeRemovals(file) } // pruneImports will remove any unused imports from the file. @@ -412,43 +372,80 @@ func pruneImports(fileSet *token.FileSet, file *ast.File) { return len(unused) > 0 }), file) - if len(unused) > 0 { - commentMap := ast.NewCommentMap(fileSet, file, file.Comments) + if len(unused) == 0 { + return + } - // Remove all import specifications - isUnusedSpec := map[*ast.ImportSpec]bool{} - for _, index := range unused { - isUnusedSpec[file.Imports[index]] = true + // Remove all import specifications + isUnusedSpec := map[*ast.ImportSpec]bool{} + for _, index := range unused { + isUnusedSpec[file.Imports[index]] = true + } + for _, decl := range file.Decls { + if d, ok := decl.(*ast.GenDecl); ok { + for i, spec := range d.Specs { + if other, ok := spec.(*ast.ImportSpec); ok && isUnusedSpec[other] { + d.Specs[i] = nil + } + } } - for i, decl := range file.Decls { - if d, ok := decl.(*ast.GenDecl); ok { - declChanged := false - for j, spec := range d.Specs { - if other, ok := spec.(*ast.ImportSpec); ok { - if isUnusedSpec[other] { + } + + // Remove the import copies in the file + for _, index := range unused { + file.Imports[index] = nil + } + + finalizeRemovals(file) +} + +// finalizeRemovals fully removes any declaration, specification, imports +// that have been set to nil. This will also remove the file's top-level +// comment group to remove any unassociated comments, including the comments +// from removed code. +func finalizeRemovals(file *ast.File) { + fileChanged := false + for i, decl := range file.Decls { + switch d := decl.(type) { + case nil: + fileChanged = true + case *ast.GenDecl: + declChanged := false + for j, spec := range d.Specs { + switch s := spec.(type) { + case nil: + declChanged = true + case *ast.ValueSpec: + specChanged := false + for _, name := range s.Names { + if name == nil { + specChanged = true + break + } + } + if specChanged { + s.Names = astutil.Squeeze(s.Names) + if len(s.Names) == 0 { declChanged = true d.Specs[j] = nil } } } - if declChanged { - d.Specs = astutil.Squeeze(d.Specs) - if len(d.Specs) == 0 { - file.Decls[i] = nil - } + } + if declChanged { + d.Specs = astutil.Squeeze(d.Specs) + if len(d.Specs) == 0 { + fileChanged = true + file.Decls[i] = nil } } } - - // Remove the import copies in the file - for _, index := range unused { - file.Imports[index] = nil - } - + } + if fileChanged { file.Decls = astutil.Squeeze(file.Decls) - file.Imports = astutil.Squeeze(file.Imports) - file.Comments = commentMap.Filter(file).Comments() } + file.Imports = astutil.Squeeze(file.Imports) + file.Comments = nil } // Options controls build process behavior. diff --git a/build/build_test.go b/build/build_test.go index a37280e6d..b0ae65d8e 100644 --- a/build/build_test.go +++ b/build/build_test.go @@ -191,6 +191,50 @@ func TestOverlayAugmentation(t *testing.T) { want: `import "math" const Tau = math.Pi * 2.0`, + expInfo: map[string]overrideInfo{ + `Foo`: {purgeMethods: true}, + `Tau`: {}, + }, + }, { + desc: `purge whole type decl`, + src: `//gopherjs:purge + type ( + Foo struct {} + bar interface{} + bob int + )`, + want: ``, + expInfo: map[string]overrideInfo{ + `Foo`: {purgeMethods: true}, + `bar`: {purgeMethods: true}, + `bob`: {purgeMethods: true}, + }, + }, { + desc: `purge part of type decl`, + src: `type ( + Foo struct {} + + //gopherjs:purge + bar interface{} + + //gopherjs:purge + bob int + )`, + want: `type ( + Foo struct {} + )`, + expInfo: map[string]overrideInfo{ + `Foo`: {}, + `bar`: {purgeMethods: true}, + `bob`: {purgeMethods: true}, + }, + }, { + desc: `purge all of a type decl`, + src: `type ( + //gopherjs:purge + Foo struct {} + )`, + want: ``, expInfo: map[string]overrideInfo{ `Foo`: {purgeMethods: true}, }, diff --git a/compiler/astutil/astutil.go b/compiler/astutil/astutil.go index 9f4bae215..98e35d872 100644 --- a/compiler/astutil/astutil.go +++ b/compiler/astutil/astutil.go @@ -193,7 +193,7 @@ func KeepOriginal(d any) bool { } // Purge returns true if gopherjs:purge directive is present -// on a struct, interface, variable, constant, or function. +// on a struct, interface, type, variable, constant, or function. // // `//gopherjs:purge` is a GopherJS-specific directive, which can be // applied in native overlays and will instruct the augmentation logic to @@ -201,7 +201,7 @@ func KeepOriginal(d any) bool { // can be used to remove code that would be invalid in GopherJS, such as code // using unsupported features (e.g. generic interfaces before generics were // fully supported). It should be used with caution since it may remove needed -// dependencies. If a struct is purged, all methods using that struct as +// dependencies. If a type is purged, all methods using that type as // a receiver will also be purged. func Purge(d any) bool { return hasDirective(d, `purge`) @@ -289,9 +289,9 @@ func NewCallbackVisitor(predicate func(node ast.Node) bool) *CallbackVisitor { } func (v *CallbackVisitor) Visit(node ast.Node) ast.Visitor { - if v.predicate == nil || node == nil || !v.predicate(node) { - v.predicate = nil - return nil + if v.predicate != nil && v.predicate(node) { + return v } - return v + v.predicate = nil + return nil } From 2d766214a2995300d23b606eac6e551ebf3a3fc2 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Wed, 13 Dec 2023 18:38:40 -0700 Subject: [PATCH 10/11] testing build updates --- build/build.go | 34 +++++++- build/build_test.go | 206 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 237 insertions(+), 3 deletions(-) diff --git a/build/build.go b/build/build.go index dcb531b7c..ccacee90c 100644 --- a/build/build.go +++ b/build/build.go @@ -339,9 +339,36 @@ func augmentOriginalFile(fileSet *token.FileSet, file *ast.File, overrides map[s d.Specs[j] = nil } case *ast.ValueSpec: - for k, name := range s.Names { - if _, ok := overrides[name.Name]; ok { - s.Names[k] = nil + if len(s.Names) == len(s.Values) { + // multi-value context + // e.g. var a, b = 2, 3 + for k, name := range s.Names { + if _, ok := overrides[name.Name]; ok { + s.Names[k] = nil + s.Values[k] = nil + } + } + } else { + // single-value context + // e.g. var a, b = func() (int, int) { return 2, 3 }() + nameRemoved := false + for _, name := range s.Names { + if _, ok := overrides[name.Name]; ok { + nameRemoved = true + name.Name = `_` + } + } + if nameRemoved { + removeSpec := true + for _, name := range s.Names { + if name.Name != `_` { + removeSpec = false + break + } + } + if removeSpec { + d.Specs[j] = nil + } } } } @@ -425,6 +452,7 @@ func finalizeRemovals(file *ast.File) { } if specChanged { s.Names = astutil.Squeeze(s.Names) + s.Values = astutil.Squeeze(s.Values) if len(s.Names) == 0 { declChanged = true d.Specs[j] = nil diff --git a/build/build_test.go b/build/build_test.go index b0ae65d8e..e58be285b 100644 --- a/build/build_test.go +++ b/build/build_test.go @@ -238,6 +238,45 @@ func TestOverlayAugmentation(t *testing.T) { expInfo: map[string]overrideInfo{ `Foo`: {purgeMethods: true}, }, + }, { + desc: `remove and purge values`, + src: `import "time" + + const ( + foo = 42 + //gopherjs:purge + bar = "gopherjs" + ) + + //gopherjs:purge + var now = time.Now`, + want: `const ( + foo = 42 + )`, + expInfo: map[string]overrideInfo{ + `foo`: {}, + `bar`: {}, + `now`: {}, + }, + }, { + desc: `purge generics`, + src: `import "cmp" + + //gopherjs:purge + type Pointer[T any] struct {} + + //gopherjs:purge + func Sort[S ~[]E, E cmp.Ordered](x S) {} + + // stub for "func Equal[S ~[]E, E any](s1, s2 S) bool" + func Equal() {}`, + want: `// stub for "func Equal[S ~[]E, E any](s1, s2 S) bool" + func Equal() {}`, + expInfo: map[string]overrideInfo{ + `Pointer`: {purgeMethods: true}, + `Sort`: {}, + `Equal`: {}, + }, }, } @@ -281,5 +320,172 @@ func TestOverlayAugmentation(t *testing.T) { } }) } +} + +func TestOriginalAugmentation(t *testing.T) { + tests := []struct { + desc string + info map[string]overrideInfo + src string + want string + }{ + { + desc: `prune an unused import`, + info: map[string]overrideInfo{}, + src: `import foo "some/other/bar"`, + want: ``, + }, { + desc: `do not affect function`, + info: map[string]overrideInfo{}, + src: `func Foo(a, b int) int { + return a + b + }`, + want: `func Foo(a, b int) int { + return a + b + }`, + }, { + desc: `change unnamed sync import`, + info: map[string]overrideInfo{}, + src: `import "sync" + + var _ = &sync.Mutex{}`, + want: `import sync "github.com/gopherjs/gopherjs/nosync" + + var _ = &sync.Mutex{}`, + }, { + desc: `change named sync import`, + info: map[string]overrideInfo{}, + src: `import foo "sync" + + var _ = &foo.Mutex{}`, + want: `import foo "github.com/gopherjs/gopherjs/nosync" + + var _ = &foo.Mutex{}`, + }, { + desc: `remove function`, + info: map[string]overrideInfo{ + `Foo`: {}, + }, + src: `func Foo(a, b int) int { + return a + b + }`, + want: ``, + }, { + desc: `keep original function`, + info: map[string]overrideInfo{ + `Foo`: {keepOriginal: true}, + }, + src: `func Foo(a, b int) int { + return a + b + }`, + want: `func _gopherjs_original_Foo(a, b int) int { + return a + b + }`, + }, { + desc: `remove types and values`, + info: map[string]overrideInfo{ + `Foo`: {}, + `now`: {}, + `bar1`: {}, + }, + src: `import "time" + + type Foo interface{} + var now = time.Now + const bar1, bar2 = 21, 42`, + want: `const bar2 = 42`, + }, { + desc: `remove in multi-value context`, + info: map[string]overrideInfo{ + `bar`: {}, + }, + src: `const foo, bar = func() (int, int) { + return 24, 12 + }()`, + want: `const foo, _ = func() (int, int) { + return 24, 12 + }()`, + }, { + desc: `full remove in multi-value context`, + info: map[string]overrideInfo{ + `bar`: {}, + }, + src: `const _, bar = func() (int, int) { + return 24, 12 + }()`, + want: ``, + }, { + desc: `purge struct and methods`, + info: map[string]overrideInfo{ + `Foo`: {purgeMethods: true}, + }, + src: `type Foo struct{ + bar int + } + + func (f Foo) GetBar() int { + return f.bar + } + + func (f *Foo) SetBar(bar int) { + f.bar = bar + } + + func NewFoo(bar int) *Foo { + return &Foo{bar: bar} + }`, + // NewFoo is not removed automatically since + // only functions with Foo as a receiver is removed. + want: `func NewFoo(bar int) *Foo { + return &Foo{bar: bar} + }`, + }, { + desc: `purge generics`, + info: map[string]overrideInfo{ + `Pointer`: {purgeMethods: true}, + `Sort`: {}, + `Equal`: {}, + }, + src: `import "cmp" + + type Pointer[T any] struct {} + func (x *Pointer[T]) Load() *T {} + func (x *Pointer[T]) Store(val *T) {} + + func Sort[S ~[]E, E cmp.Ordered](x S) {} + + // overlay had stub "func Equal() {}" + func Equal[S ~[]E, E any](s1, s2 S) bool {}`, + want: ``, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + pkgName := "package testpackage\n\n" + importPath := `math/rand` + fsetSrc := token.NewFileSet() + fileSrc := srctesting.Parse(t, fsetSrc, pkgName+test.src) + + augmentOriginalImports(importPath, fileSrc) + augmentOriginalFile(fsetSrc, fileSrc, test.info) + pruneImports(fsetSrc, fileSrc) + + buf := &bytes.Buffer{} + _ = printer.Fprint(buf, fsetSrc, fileSrc) + got := buf.String() + fsetWant := token.NewFileSet() + fileWant := srctesting.Parse(t, fsetWant, pkgName+test.want) + + buf.Reset() + _ = printer.Fprint(buf, fsetWant, fileWant) + want := buf.String() + + if got != want { + t.Errorf("augmentOriginalImports, augmentOriginalFile, and pruneImports got unexpected code:\n"+ + "returned:\n\t%q\nwant:\n\t%q", got, want) + } + }) + } } From 959c24bc7bbf5cf45656b2942dbc552c724ebba9 Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Mon, 18 Dec 2023 10:07:43 -0700 Subject: [PATCH 11/11] about to break up into smaller tickets --- build/build_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/build/build_test.go b/build/build_test.go index e58be285b..8b00336bd 100644 --- a/build/build_test.go +++ b/build/build_test.go @@ -258,6 +258,32 @@ func TestOverlayAugmentation(t *testing.T) { `bar`: {}, `now`: {}, }, + }, { + desc: `imports not confused by local variables`, + src: `import ( + "cmp" + "time" + ) + + //gopherjs:purge + func Sort[S ~[]E, E cmp.Ordered](x S) {} + + func SecondsSince(start time.Time) int { + cmp := time.Now().Sub(start) + return int(cmp.Second()) + }`, + want: `import ( + "time" + ) + + func SecondsSince(start time.Time) int { + cmp := time.Now().Sub(start) + return int(cmp.Second()) + }`, + expInfo: map[string]overrideInfo{ + `Sort`: {}, + `SecondsSince`: {}, + }, }, { desc: `purge generics`, src: `import "cmp"