Skip to content

Support embed package. #1145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 26 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
61801ed
compiler/natives/src: add embed
visualfc Sep 9, 2022
b0db6f5
compiler/natives: go generate
visualfc Sep 9, 2022
522585d
build: check and generate embed file for build/test
visualfc Sep 9, 2022
561171d
build: BuildFiles support check embed file for build/run
visualfc Sep 10, 2022
7dd9288
tests/testdata/legacy_syscall: fix go:build
visualfc Sep 10, 2022
2a8f084
goembed v0.2.3
visualfc Sep 11, 2022
092a24b
checkEmbed: check load file error
visualfc Sep 11, 2022
4316b6d
build: move checkEmbed to BuildPackage
visualfc Sep 11, 2022
b4fad18
build: checkEmbed unknown type check error
visualfc Sep 11, 2022
7c483c8
build: embedFiles
visualfc Sep 12, 2022
aec0b57
compiler/natives/src: prealloc embed
visualfc Sep 12, 2022
7cb07f3
build: embedFiles rewrite ast for go:embed vars
visualfc Sep 12, 2022
f4c2418
compiler/natives/src: add embed/internal/embedtest
visualfc Sep 12, 2022
0c19332
build: BuildFiles ctx.ReadDir prealloc
visualfc Sep 12, 2022
625d4cc
build: joinEmbedPatternPos
visualfc Sep 12, 2022
1e0f65a
goembed v0.2.5
visualfc Sep 13, 2022
a80e720
goembed v0.2.7
visualfc Sep 14, 2022
2a6f9de
goembed v0.2.8
visualfc Sep 15, 2022
fd891b2
goembed v0.3.0 go:embed check multiple file error if kind not EmbedFiles
visualfc Sep 15, 2022
ffe4555
goembed v0.3.1
visualfc Sep 16, 2022
c5140d5
build: embedFiles check goembed.EmbedMaybeAlias
visualfc Sep 16, 2022
b612190
x
visualfc Sep 16, 2022
a9d764a
goembed v0.3.2
visualfc Sep 16, 2022
b799041
Merge remote-tracking branch 'upstream/master' into embed
visualfc Sep 19, 2022
8fb8e58
goembed v0.3.3
visualfc Sep 19, 2022
c552a3d
remove compiler/natives/src/embed/internal/embedtest
visualfc Sep 19, 2022
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .std_test_pkg_exclusions
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
embed/internal/embedtest
encoding/xml
go/build
go/internal/srcimporter
Expand Down
59 changes: 43 additions & 16 deletions build/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"go/scanner"
"go/token"
"go/types"
"io/fs"
"os"
"os/exec"
"path"
Expand Down Expand Up @@ -398,11 +399,12 @@ func (p *PackageData) InternalBuildContext() *build.Context {
func (p *PackageData) TestPackage() *PackageData {
return &PackageData{
Package: &build.Package{
Name: p.Name,
ImportPath: p.ImportPath,
Dir: p.Dir,
GoFiles: append(p.GoFiles, p.TestGoFiles...),
Imports: append(p.Imports, p.TestImports...),
Name: p.Name,
ImportPath: p.ImportPath,
Dir: p.Dir,
GoFiles: append(p.GoFiles, p.TestGoFiles...),
Imports: append(p.Imports, p.TestImports...),
EmbedPatternPos: joinEmbedPatternPos(p.EmbedPatternPos, p.TestEmbedPatternPos),
},
IsTest: true,
JSFiles: p.JSFiles,
Expand All @@ -414,11 +416,12 @@ func (p *PackageData) TestPackage() *PackageData {
func (p *PackageData) XTestPackage() *PackageData {
return &PackageData{
Package: &build.Package{
Name: p.Name + "_test",
ImportPath: p.ImportPath + "_test",
Dir: p.Dir,
GoFiles: p.XTestGoFiles,
Imports: p.XTestImports,
Name: p.Name + "_test",
ImportPath: p.ImportPath + "_test",
Dir: p.Dir,
GoFiles: p.XTestGoFiles,
Imports: p.XTestImports,
EmbedPatternPos: p.XTestEmbedPatternPos,
},
IsTest: true,
bctx: p.bctx,
Expand Down Expand Up @@ -548,12 +551,30 @@ func (s *Session) BuildFiles(filenames []string, pkgObj string, cwd string) erro
return fmt.Errorf("named files must all be in one directory; have: %v", strings.Join(dirList, ", "))
}

root := dirList[0]
ctx := build.Default
ctx.UseAllFiles = true
ctx.ReadDir = func(dir string) ([]fs.FileInfo, error) {
n := len(filenames)
infos := make([]fs.FileInfo, n)
for i := 0; i < n; i++ {
info, err := os.Stat(filenames[i])
if err != nil {
return nil, err
}
infos[i] = info
}
return infos, nil
}
p, err := ctx.Import(".", root, 0)
if err != nil {
return err
}
p.Name = "main"
p.ImportPath = "main"

pkg := &PackageData{
Package: &build.Package{
Name: "main",
ImportPath: "main",
Dir: dirList[0],
},
Package: p,
// This ephemeral package doesn't have a unique import path to be used as a
// build cache key, so we never cache it.
SrcModTime: time.Now().Add(time.Hour),
Expand All @@ -562,7 +583,6 @@ func (s *Session) BuildFiles(filenames []string, pkgObj string, cwd string) erro

for _, file := range filenames {
if !strings.HasSuffix(file, ".inc.js") {
pkg.GoFiles = append(pkg.GoFiles, filepath.Base(file))
continue
}

Expand Down Expand Up @@ -676,6 +696,13 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
if err != nil {
return nil, err
}
embed, err := embedFiles(pkg, fileSet, files)
if err != nil {
return nil, err
}
if embed != nil {
files = append(files, embed)
}

importContext := &compiler.ImportContext{
Packages: s.Types,
Expand Down
175 changes: 175 additions & 0 deletions build/embed.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
package build

import (
"bytes"
"fmt"
"go/ast"
"go/parser"
"go/token"
"strconv"

"github.com/visualfc/goembed"
)

func buildIdent(name string) string {
return fmt.Sprintf("__gopherjs_embed_%x__", name)
}

var embed_head = `package %v

import (
"embed"
_ "unsafe"
)

//go:linkname __gopherjs_embed_buildFS__ embed.buildFS
func __gopherjs_embed_buildFS__(list []struct {
name string
data string
hash [16]byte
}) (f embed.FS)
`

// embedFiles generates an additional source file, which initializes all variables in the package with a go:embed directive.
func embedFiles(pkg *PackageData, fset *token.FileSet, files []*ast.File) (*ast.File, error) {
if len(pkg.EmbedPatternPos) == 0 {
return nil, nil
}

ems, err := goembed.CheckEmbed(pkg.EmbedPatternPos, fset, files)
if err != nil {
return nil, err
}

r := goembed.NewResolve()
for _, em := range ems {
fs, err := r.Load(pkg.Dir, fset, em)
if err != nil {
return nil, err
}
switch em.Kind {
case goembed.EmbedMaybeAlias:
// value = Type(data)
// valid alias string or []byte type used by types.check
em.Spec.Values = []ast.Expr{
&ast.CallExpr{
Fun: em.Spec.Type,
Args: []ast.Expr{
&ast.Ident{Name: buildIdent(fs[0].Name),
NamePos: em.Spec.Names[0].NamePos},
},
}}
case goembed.EmbedBytes:
// value = []byte(data)
em.Spec.Values = []ast.Expr{
&ast.CallExpr{
Fun: em.Spec.Type,
Args: []ast.Expr{ast.NewIdent(buildIdent(fs[0].Name))},
}}
case goembed.EmbedString:
// value = data
em.Spec.Values = []ast.Expr{ast.NewIdent(buildIdent(fs[0].Name))}
case goembed.EmbedFiles:
// value = __gopherjs_embed_buildFS__([]struct{name string; data string; hash [16]byte}{...})
fs = goembed.BuildFS(fs)
elts := make([]ast.Expr, len(fs))
for i, f := range fs {
if len(f.Data) == 0 {
elts[i] = &ast.CompositeLit{
Elts: []ast.Expr{
&ast.BasicLit{Kind: token.STRING, Value: strconv.Quote(f.Name)},
&ast.BasicLit{Kind: token.STRING, Value: `""`},
&ast.CompositeLit{
Type: &ast.ArrayType{
Len: &ast.BasicLit{Kind: token.INT, Value: "16"},
Elt: ast.NewIdent("byte"),
},
},
},
}
} else {
var hash [16]ast.Expr
for j, v := range f.Hash {
hash[j] = &ast.BasicLit{Kind: token.INT, Value: strconv.Itoa(int(v))}
}
elts[i] = &ast.CompositeLit{
Elts: []ast.Expr{
&ast.BasicLit{Kind: token.STRING, Value: strconv.Quote(f.Name)},
ast.NewIdent(buildIdent(f.Name)),
&ast.CompositeLit{
Type: &ast.ArrayType{
Len: &ast.BasicLit{Kind: token.INT, Value: "16"},
Elt: ast.NewIdent("byte"),
},
Elts: hash[:],
},
},
}
}
}
call := &ast.CallExpr{
Fun: ast.NewIdent("__gopherjs_embed_buildFS__"),
Args: []ast.Expr{
&ast.CompositeLit{
Type: &ast.ArrayType{
Elt: &ast.StructType{
Fields: &ast.FieldList{
List: []*ast.Field{
&ast.Field{
Names: []*ast.Ident{ast.NewIdent("name")},
Type: ast.NewIdent("string"),
},
&ast.Field{
Names: []*ast.Ident{ast.NewIdent("data")},
Type: ast.NewIdent("string"),
},
&ast.Field{
Names: []*ast.Ident{ast.NewIdent("hash")},
Type: &ast.ArrayType{
Len: &ast.BasicLit{Kind: token.INT, Value: "16"},
Elt: ast.NewIdent("byte"),
},
},
},
},
},
},
Elts: elts,
},
},
}
em.Spec.Values = []ast.Expr{call}
}
}

var buf bytes.Buffer
fmt.Fprintf(&buf, embed_head, pkg.Name)
buf.WriteString("\nconst (\n")
for _, f := range r.Files() {
if len(f.Data) == 0 {
fmt.Fprintf(&buf, "\t%v = \"\"\n", buildIdent(f.Name))
} else {
fmt.Fprintf(&buf, "\t%v = \"%v\"\n", buildIdent(f.Name), goembed.BytesToHex(f.Data))
}
}
buf.WriteString(")\n\n")
f, err := parser.ParseFile(fset, "js_embed.go", buf.String(), parser.ParseComments)
if err != nil {
return nil, err
}
return f, nil
}

func joinEmbedPatternPos(m1, m2 map[string][]token.Position) map[string][]token.Position {
if len(m1) == 0 && len(m2) == 0 {
return nil
}
m := make(map[string][]token.Position)
for k, v := range m1 {
m[k] = v
}
for k, v := range m2 {
m[k] = append(m[k], v...)
}
return m
}
Loading