Skip to content

Embed core GopherJS packages into build system; enable vendoring of GopherJS. #787

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Apr 21, 2018
Merged

Embed core GopherJS packages into build system; enable vendoring of GopherJS. #787

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
f681903
build: Make fewer copies of NewBuildContext.
dmitshur Apr 7, 2018
a4af087
compiler/gopherjspkg: Add package.
dmitshur Apr 7, 2018
c121b3d
build: Load js, nosync packages from gopherjspkg rather than GOPATH.
dmitshur Apr 7, 2018
b90dbcb
build, compiler/typesutil: Don't use vendor directory to resolve js, …
dmitshur Apr 7, 2018
b24e356
Restore support for testing js package.
dmitshur Apr 8, 2018
d0d69c0
tests: Add test that GopherJS can be vendored.
dmitshur Apr 15, 2018
c8e5f7c
README: Remove note saying vendoring GopherJS is unsupported.
dmitshur Apr 8, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ GopherJS compiles Go code ([golang.org](https://golang.org/)) to pure JavaScript
Give GopherJS a try on the [GopherJS Playground](http://gopherjs.github.io/playground/).

### What is supported?
Nearly everything, including Goroutines ([compatibility table](https://github.com/gopherjs/gopherjs/blob/master/doc/packages.md)). Performance is quite good in most cases, see [HTML5 game engine benchmark](https://ajhager.github.io/engi/demos/botmark.html). Cgo is not supported. Using a vendored copy of GopherJS is currently not supported, see [#415](https://github.com/gopherjs/gopherjs/issues/415).
Nearly everything, including Goroutines ([compatibility table](https://github.com/gopherjs/gopherjs/blob/master/doc/packages.md)). Performance is quite good in most cases, see [HTML5 game engine benchmark](https://ajhager.github.io/engi/demos/botmark.html). Cgo is not supported.

### Installation and Usage
Get or update GopherJS and dependencies with:
Expand Down
105 changes: 83 additions & 22 deletions build/build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,11 @@ import (

"github.com/fsnotify/fsnotify"
"github.com/gopherjs/gopherjs/compiler"
"github.com/gopherjs/gopherjs/compiler/gopherjspkg"
"github.com/gopherjs/gopherjs/compiler/natives"
"github.com/neelance/sourcemap"
"github.com/shurcooL/httpfs/vfsutil"
"golang.org/x/tools/go/buildutil"
)

type ImportCError struct {
Expand All @@ -33,7 +36,13 @@ func (e *ImportCError) Error() string {
return e.pkgPath + `: importing "C" is not supported by GopherJS`
}

// NewBuildContext creates a build context for building Go packages
// with GopherJS compiler.
//
// Core GopherJS packages (i.e., "github.com/gopherjs/gopherjs/js", "github.com/gopherjs/gopherjs/nosync")
// are loaded from gopherjspkg.FS virtual filesystem rather than GOPATH.
func NewBuildContext(installSuffix string, buildTags []string) *build.Context {
gopherjsRoot := filepath.Join(build.Default.GOROOT, "src", "github.com", "gopherjs", "gopherjs")
return &build.Context{
GOROOT: build.Default.GOROOT,
GOPATH: build.Default.GOPATH,
Expand All @@ -47,9 +56,52 @@ func NewBuildContext(installSuffix string, buildTags []string) *build.Context {
),
ReleaseTags: build.Default.ReleaseTags,
CgoEnabled: true, // detect `import "C"` to throw proper error

IsDir: func(path string) bool {
if strings.HasPrefix(path, gopherjsRoot+string(filepath.Separator)) {
path = filepath.ToSlash(path[len(gopherjsRoot):])
if fi, err := vfsutil.Stat(gopherjspkg.FS, path); err == nil {
return fi.IsDir()
}
}
fi, err := os.Stat(path)
return err == nil && fi.IsDir()
},
ReadDir: func(path string) ([]os.FileInfo, error) {
if strings.HasPrefix(path, gopherjsRoot+string(filepath.Separator)) {
path = filepath.ToSlash(path[len(gopherjsRoot):])
if fis, err := vfsutil.ReadDir(gopherjspkg.FS, path); err == nil {
return fis, nil
}
}
return ioutil.ReadDir(path)
},
OpenFile: func(path string) (io.ReadCloser, error) {
if strings.HasPrefix(path, gopherjsRoot+string(filepath.Separator)) {
path = filepath.ToSlash(path[len(gopherjsRoot):])
if f, err := gopherjspkg.FS.Open(path); err == nil {
return f, nil
}
}
return os.Open(path)
},
}
}

// statFile returns an os.FileInfo describing the named file.
// For files in "$GOROOT/src/github.com/gopherjs/gopherjs" directory,
// gopherjspkg.FS is consulted first.
func statFile(path string) (os.FileInfo, error) {
gopherjsRoot := filepath.Join(build.Default.GOROOT, "src", "github.com", "gopherjs", "gopherjs")
if strings.HasPrefix(path, gopherjsRoot+string(filepath.Separator)) {
path = filepath.ToSlash(path[len(gopherjsRoot):])
if fi, err := vfsutil.Stat(gopherjspkg.FS, path); err == nil {
return fi, nil
}
}
return os.Stat(path)
}

// Import returns details about the Go package named by the import path. If the
// path is a local import path naming a package that can be imported using
// a standard import path, the returned package will set p.ImportPath to
Expand All @@ -72,11 +124,13 @@ func Import(path string, mode build.ImportMode, installSuffix string, buildTags
// Import will not be able to resolve relative import paths.
wd = ""
}
return importWithSrcDir(path, wd, mode, installSuffix, buildTags)
bctx := NewBuildContext(installSuffix, buildTags)
return importWithSrcDir(*bctx, path, wd, mode, installSuffix)
}

func importWithSrcDir(path string, srcDir string, mode build.ImportMode, installSuffix string, buildTags []string) (*PackageData, error) {
bctx := NewBuildContext(installSuffix, buildTags)
func importWithSrcDir(bctx build.Context, path string, srcDir string, mode build.ImportMode, installSuffix string) (*PackageData, error) {
// bctx is passed by value, so it can be modified here.
var isVirtual bool
switch path {
case "syscall":
// syscall needs to use a typical GOARCH like amd64 to pick up definitions for _Socklen, BpfInsn, IFNAMSIZ, Timeval, BpfStat, SYS_FCNTL, Flock_t, etc.
Expand All @@ -91,17 +145,17 @@ func importWithSrcDir(path string, srcDir string, mode build.ImportMode, install
case "crypto/x509", "os/user":
// These stdlib packages have cgo and non-cgo versions (via build tags); we want the latter.
bctx.CgoEnabled = false
case "github.com/gopherjs/gopherjs/js", "github.com/gopherjs/gopherjs/nosync":
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Isn't it ok to check in the above way like strings.HasPrefix(path, gopherJSRoot+string(filepath.Separator) for consistency?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, because this needs to be those 2 packages only. We don't want github.com/gopherjs/gopherjs/compiler or others to be included.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, thanks. I feel like this set is duplicated with FS definition in fs.go. Would it be possible to unify these sets into one?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The "core" GopherJS packages are enumerated and documented in the package comment of ./compiler/gopherjspkg package (see compiler/gopherjspkg/doc.go file).

Here, the ./build package implements that. It uses import paths.

In gopherjspkg.FS, the code picks up those packages via relative directory paths.

I don't think anything more needs to be done, this is very simple, and trying to make it more DRY isn't worth the effort (I'm usually all for it, but it needs to be worthwhile).

// These packages are already embedded via gopherjspkg.FS virtual filesystem (which can be
// safely vendored). Don't try to use vendor directory to resolve them.
mode |= build.IgnoreVendor
isVirtual = true
}
pkg, err := bctx.Import(path, srcDir, mode)
if err != nil {
return nil, err
}

// TODO: Resolve issue #415 and remove this temporary workaround.
if strings.HasSuffix(pkg.ImportPath, "/vendor/github.com/gopherjs/gopherjs/js") {
return nil, fmt.Errorf("vendoring github.com/gopherjs/gopherjs/js package is not supported, see https://github.com/gopherjs/gopherjs/issues/415")
}

switch path {
case "os":
pkg.GoFiles = excludeExecutable(pkg.GoFiles) // Need to exclude executable implementation files, because some of them contain package scope variables that perform (indirectly) syscalls on init.
Expand Down Expand Up @@ -135,12 +189,12 @@ func importWithSrcDir(path string, srcDir string, mode build.ImportMode, install
}
}

jsFiles, err := jsFilesFromDir(pkg.Dir)
jsFiles, err := jsFilesFromDir(&bctx, pkg.Dir)
if err != nil {
return nil, err
}

return &PackageData{Package: pkg, JSFiles: jsFiles}, nil
return &PackageData{Package: pkg, JSFiles: jsFiles, IsVirtual: isVirtual}, nil
}

// excludeExecutable excludes all executable implementation .go files.
Expand Down Expand Up @@ -174,12 +228,13 @@ Outer:
// ImportDir is like Import but processes the Go package found in the named
// directory.
func ImportDir(dir string, mode build.ImportMode, installSuffix string, buildTags []string) (*PackageData, error) {
pkg, err := NewBuildContext(installSuffix, buildTags).ImportDir(dir, mode)
bctx := NewBuildContext(installSuffix, buildTags)
pkg, err := bctx.ImportDir(dir, mode)
if err != nil {
return nil, err
}

jsFiles, err := jsFilesFromDir(pkg.Dir)
jsFiles, err := jsFilesFromDir(bctx, pkg.Dir)
if err != nil {
return nil, err
}
Expand All @@ -198,7 +253,7 @@ func ImportDir(dir string, mode build.ImportMode, installSuffix string, buildTag
// as an existing file from the standard library). For all identifiers that exist
// in the original AND the overrides, the original identifier in the AST gets
// replaced by `_`. New identifiers that don't exist in original package get added.
func parseAndAugment(pkg *build.Package, isTest bool, fileSet *token.FileSet) ([]*ast.File, error) {
func parseAndAugment(bctx *build.Context, pkg *build.Package, isTest bool, fileSet *token.FileSet) ([]*ast.File, error) {
var files []*ast.File
replacedDeclNames := make(map[string]bool)
funcName := func(d *ast.FuncDecl) string {
Expand Down Expand Up @@ -302,10 +357,10 @@ func parseAndAugment(pkg *build.Package, isTest bool, fileSet *token.FileSet) ([

var errList compiler.ErrorList
for _, name := range pkg.GoFiles {
if !filepath.IsAbs(name) {
if !filepath.IsAbs(name) { // name might be absolute if specified directly. E.g., `gopherjs build /abs/file.go`.
name = filepath.Join(pkg.Dir, name)
}
r, err := os.Open(name)
r, err := buildutil.OpenFile(bctx, name)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -406,10 +461,12 @@ type PackageData struct {
IsTest bool // IsTest is true if the package is being built for running tests.
SrcModTime time.Time
UpToDate bool
IsVirtual bool // If true, the package does not have a corresponding physical directory on disk.
}

type Session struct {
options *Options
bctx *build.Context
Archives map[string]*compiler.Archive
Types map[string]*types.Package
Watcher *fsnotify.Watcher
Expand All @@ -428,6 +485,7 @@ func NewSession(options *Options) *Session {
options: options,
Archives: make(map[string]*compiler.Archive),
}
s.bctx = NewBuildContext(s.InstallSuffix(), s.options.BuildTags)
s.Types = make(map[string]*types.Package)
if options.Watch {
if out, err := exec.Command("ulimit", "-n").Output(); err == nil {
Expand All @@ -445,6 +503,9 @@ func NewSession(options *Options) *Session {
return s
}

// BuildContext returns the session's build context.
func (s *Session) BuildContext() *build.Context { return s.bctx }

func (s *Session) InstallSuffix() string {
if s.options.Minify {
return "min"
Expand All @@ -456,12 +517,12 @@ func (s *Session) BuildDir(packagePath string, importPath string, pkgObj string)
if s.Watcher != nil {
s.Watcher.Add(packagePath)
}
buildPkg, err := NewBuildContext(s.InstallSuffix(), s.options.BuildTags).ImportDir(packagePath, 0)
buildPkg, err := s.bctx.ImportDir(packagePath, 0)
if err != nil {
return err
}
pkg := &PackageData{Package: buildPkg}
jsFiles, err := jsFilesFromDir(pkg.Dir)
jsFiles, err := jsFilesFromDir(s.bctx, pkg.Dir)
if err != nil {
return err
}
Expand Down Expand Up @@ -514,7 +575,7 @@ func (s *Session) BuildImportPath(path string) (*compiler.Archive, error) {
}

func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*PackageData, *compiler.Archive, error) {
pkg, err := importWithSrcDir(path, srcDir, 0, s.InstallSuffix(), s.options.BuildTags)
pkg, err := importWithSrcDir(*s.bctx, path, srcDir, 0, s.InstallSuffix())
if s.Watcher != nil && pkg != nil { // add watch even on error
s.Watcher.Add(pkg.Dir)
}
Expand Down Expand Up @@ -580,7 +641,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
}

for _, name := range append(pkg.GoFiles, pkg.JSFiles...) {
fileInfo, err := os.Stat(filepath.Join(pkg.Dir, name))
fileInfo, err := statFile(filepath.Join(pkg.Dir, name))
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -614,7 +675,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
}

fileSet := token.NewFileSet()
files, err := parseAndAugment(pkg.Package, pkg.IsTest, fileSet)
files, err := parseAndAugment(s.bctx, pkg.Package, pkg.IsTest, fileSet)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -752,8 +813,8 @@ func NewMappingCallback(m *sourcemap.Map, goroot, gopath string, localMap bool)
}
}

func jsFilesFromDir(dir string) ([]string, error) {
files, err := ioutil.ReadDir(dir)
func jsFilesFromDir(bctx *build.Context, dir string) ([]string, error) {
files, err := buildutil.ReadDir(bctx, dir)
if err != nil {
return nil, err
}
Expand Down
6 changes: 3 additions & 3 deletions build/build_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ func TestNativesDontImportExtraPackages(t *testing.T) {

// Use parseAndAugment to get a list of augmented AST files.
fset := token.NewFileSet()
files, err := parseAndAugment(bpkg, false, fset)
files, err := parseAndAugment(NewBuildContext("", nil), bpkg, false, fset)
if err != nil {
t.Fatalf("github.com/gopherjs/gopherjs/build.parseAndAugment: %v", err)
}
Expand Down Expand Up @@ -116,7 +116,7 @@ func TestNativesDontImportExtraPackages(t *testing.T) {

// Use parseAndAugment to get a list of augmented AST files.
fset := token.NewFileSet()
files, err := parseAndAugment(bpkg, true, fset)
files, err := parseAndAugment(NewBuildContext("", nil), bpkg, true, fset)
if err != nil {
t.Fatalf("github.com/gopherjs/gopherjs/build.parseAndAugment: %v", err)
}
Expand Down Expand Up @@ -158,7 +158,7 @@ func TestNativesDontImportExtraPackages(t *testing.T) {

// Use parseAndAugment to get a list of augmented AST files, then check only the external test files.
fset := token.NewFileSet()
files, err := parseAndAugment(bpkg, true, fset)
files, err := parseAndAugment(NewBuildContext("", nil), bpkg, true, fset)
if err != nil {
t.Fatalf("github.com/gopherjs/gopherjs/build.parseAndAugment: %v", err)
}
Expand Down
12 changes: 12 additions & 0 deletions compiler/gopherjspkg/doc.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
// Package gopherjspkg provides core GopherJS packages via a virtual filesystem.
//
// Core GopherJS packages are packages that are critical for GopherJS compiler
// operation. They are needed to build the Go standard library with GopherJS.
// Currently, they include:
//
// github.com/gopherjs/gopherjs/js
// github.com/gopherjs/gopherjs/nosync
//
package gopherjspkg

//go:generate vfsgendev -source="github.com/gopherjs/gopherjs/compiler/gopherjspkg".FS -tag=gopherjsdev
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Where does vfgendev come from?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See https://github.com/gopherjs/gopherjs/wiki/Developer-Guidelines#code-generation.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please can we make this go:generate directive not rely on vfsgendev being in PATH?

Copy link
Member Author

@dmitshur dmitshur Apr 20, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's out of scope for this PR. This gopherjspkg package is the same in structure as natives.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point. This would be better solved in any case by golang/go#22726

31 changes: 31 additions & 0 deletions compiler/gopherjspkg/fs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// +build gopherjsdev

package gopherjspkg

import (
"go/build"
"log"
"net/http"
"os"
pathpkg "path"

"github.com/shurcooL/httpfs/filter"
)

// FS is a virtual filesystem that contains core GopherJS packages.
var FS = filter.Keep(
http.Dir(importPathToDir("github.com/gopherjs/gopherjs")),
func(path string, fi os.FileInfo) bool {
return path == "/" ||
path == "/js" || (pathpkg.Dir(path) == "/js" && !fi.IsDir()) ||
path == "/nosync" || (pathpkg.Dir(path) == "/nosync" && !fi.IsDir())
},
)

func importPathToDir(importPath string) string {
p, err := build.Import(importPath, "", build.FindOnly)
if err != nil {
log.Fatalln(err)
}
return p.Dir
}
Loading