diff --git a/build/build.go b/build/build.go index 1b10464fd..48e7ef7c6 100644 --- a/build/build.go +++ b/build/build.go @@ -1,3 +1,8 @@ +// Package build implements GopherJS build system. +// +// WARNING: This package's API is treated as internal and currently doesn't +// provide any API stability guarantee, use it at your own risk. If you need a +// stable interface, prefer invoking the gopherjs CLI tool as a subprocess. package build import ( @@ -8,7 +13,6 @@ import ( "go/scanner" "go/token" "go/types" - "io" "io/ioutil" "os" "os/exec" @@ -26,6 +30,8 @@ import ( "github.com/neelance/sourcemap" "github.com/shurcooL/httpfs/vfsutil" "golang.org/x/tools/go/buildutil" + + _ "github.com/gopherjs/gopherjs/build/versionhack" // go/build release tags hack. ) // DefaultGOROOT is the default GOROOT value for builds. @@ -53,52 +59,13 @@ func (e *ImportCError) Error() string { // with GopherJS compiler. // // Core GopherJS packages (i.e., "github.com/gopherjs/gopherjs/js", "github.com/gopherjs/gopherjs/nosync") -// are loaded from gopherjspkg.FS virtual filesystem rather than GOPATH. -func NewBuildContext(installSuffix string, buildTags []string) *build.Context { +// are loaded from gopherjspkg.FS virtual filesystem if not present in GOPATH or +// go.mod. +func NewBuildContext(installSuffix string, buildTags []string) XContext { gopherjsRoot := filepath.Join(DefaultGOROOT, "src", "github.com", "gopherjs", "gopherjs") - return &build.Context{ - GOROOT: DefaultGOROOT, - GOPATH: build.Default.GOPATH, - GOOS: build.Default.GOOS, - GOARCH: "js", - InstallSuffix: installSuffix, - Compiler: "gc", - BuildTags: append(buildTags, - "netgo", // See https://godoc.org/net#hdr-Name_Resolution. - "purego", // See https://golang.org/issues/23172. - "math_big_pure_go", // Use pure Go version of math/big. - ), - ReleaseTags: build.Default.ReleaseTags[:compiler.GoVersion], - CgoEnabled: true, // detect `import "C"` to throw proper error - - IsDir: func(path string) bool { - if strings.HasPrefix(path, gopherjsRoot+string(filepath.Separator)) { - path = filepath.ToSlash(path[len(gopherjsRoot):]) - if fi, err := vfsutil.Stat(gopherjspkg.FS, path); err == nil { - return fi.IsDir() - } - } - fi, err := os.Stat(path) - return err == nil && fi.IsDir() - }, - ReadDir: func(path string) ([]os.FileInfo, error) { - if strings.HasPrefix(path, gopherjsRoot+string(filepath.Separator)) { - path = filepath.ToSlash(path[len(gopherjsRoot):]) - if fis, err := vfsutil.ReadDir(gopherjspkg.FS, path); err == nil { - return fis, nil - } - } - return ioutil.ReadDir(path) - }, - OpenFile: func(path string) (io.ReadCloser, error) { - if strings.HasPrefix(path, gopherjsRoot+string(filepath.Separator)) { - path = filepath.ToSlash(path[len(gopherjsRoot):]) - if f, err := gopherjspkg.FS.Open(path); err == nil { - return f, nil - } - } - return os.Open(path) - }, + return &chainedCtx{ + primary: goCtx(installSuffix, buildTags), + secondary: embeddedCtx(&withPrefix{gopherjspkg.FS, gopherjsRoot}, installSuffix, buildTags), } } @@ -138,34 +105,12 @@ func Import(path string, mode build.ImportMode, installSuffix string, buildTags // Import will not be able to resolve relative import paths. wd = "" } - bctx := NewBuildContext(installSuffix, buildTags) - return importWithSrcDir(*bctx, path, wd, mode, installSuffix) + xctx := NewBuildContext(installSuffix, buildTags) + return importWithSrcDir(xctx, path, wd, mode, installSuffix) } -func importWithSrcDir(bctx build.Context, path string, srcDir string, mode build.ImportMode, installSuffix string) (*PackageData, error) { - // bctx is passed by value, so it can be modified here. - var isVirtual bool - switch path { - case "syscall": - // syscall needs to use a typical GOARCH like amd64 to pick up definitions for _Socklen, BpfInsn, IFNAMSIZ, Timeval, BpfStat, SYS_FCNTL, Flock_t, etc. - bctx.GOARCH = build.Default.GOARCH - bctx.InstallSuffix = "js" - if installSuffix != "" { - bctx.InstallSuffix += "_" + installSuffix - } - case "syscall/js": - // There are no buildable files in this package, but we need to use files in the virtual directory. - mode |= build.FindOnly - case "crypto/x509", "os/user": - // These stdlib packages have cgo and non-cgo versions (via build tags); we want the latter. - bctx.CgoEnabled = false - case "github.com/gopherjs/gopherjs/js", "github.com/gopherjs/gopherjs/nosync": - // These packages are already embedded via gopherjspkg.FS virtual filesystem (which can be - // safely vendored). Don't try to use vendor directory to resolve them. - mode |= build.IgnoreVendor - isVirtual = true - } - pkg, err := bctx.Import(path, srcDir, mode) +func importWithSrcDir(xctx XContext, path string, srcDir string, mode build.ImportMode, installSuffix string) (*PackageData, error) { + pkg, err := xctx.Import(path, srcDir, mode) if err != nil { return nil, err } @@ -181,7 +126,7 @@ func importWithSrcDir(bctx build.Context, path string, srcDir string, mode build case "runtime": pkg.GoFiles = []string{} // Package sources are completely replaced in natives. case "runtime/internal/sys": - pkg.GoFiles = []string{fmt.Sprintf("zgoos_%s.go", bctx.GOOS), "zversion.go"} + pkg.GoFiles = []string{fmt.Sprintf("zgoos_%s.go", xctx.GOOS()), "zversion.go"} case "runtime/pprof": pkg.GoFiles = nil case "internal/poll": @@ -201,7 +146,7 @@ func importWithSrcDir(bctx build.Context, path string, srcDir string, mode build // Just like above, https://github.com/gopherjs/gopherjs/issues/693 is // probably the best long-term option. pkg.GoFiles = include( - exclude(pkg.GoFiles, fmt.Sprintf("root_%s.go", bctx.GOOS)), + exclude(pkg.GoFiles, fmt.Sprintf("root_%s.go", xctx.GOOS())), "root_unix.go", "root_js.go") case "syscall/js": // Reuse upstream tests to ensure conformance, but completely replace @@ -226,12 +171,7 @@ func importWithSrcDir(bctx build.Context, path string, srcDir string, mode build } } - jsFiles, err := jsFilesFromDir(&bctx, pkg.Dir) - if err != nil { - return nil, err - } - - return &PackageData{Package: pkg, JSFiles: jsFiles, IsVirtual: isVirtual}, nil + return pkg, nil } // excludeExecutable excludes all executable implementation .go files. @@ -271,18 +211,13 @@ func include(files []string, includes ...string) []string { // ImportDir is like Import but processes the Go package found in the named // directory. func ImportDir(dir string, mode build.ImportMode, installSuffix string, buildTags []string) (*PackageData, error) { - bctx := NewBuildContext(installSuffix, buildTags) - pkg, err := bctx.ImportDir(dir, mode) + xctx := NewBuildContext(installSuffix, buildTags) + pkg, err := xctx.Import(".", dir, mode) if err != nil { return nil, err } - jsFiles, err := jsFilesFromDir(bctx, pkg.Dir) - if err != nil { - return nil, err - } - - return &PackageData{Package: pkg, JSFiles: jsFiles}, nil + return pkg, nil } // parseAndAugment parses and returns all .go files of given pkg. @@ -296,7 +231,7 @@ func ImportDir(dir string, mode build.ImportMode, installSuffix string, buildTag // as an existing file from the standard library). For all identifiers that exist // in the original AND the overrides, the original identifier in the AST gets // replaced by `_`. New identifiers that don't exist in original package get added. -func parseAndAugment(bctx *build.Context, pkg *build.Package, isTest bool, fileSet *token.FileSet) ([]*ast.File, error) { +func parseAndAugment(xctx XContext, pkg *PackageData, isTest bool, fileSet *token.FileSet) ([]*ast.File, error) { var files []*ast.File replacedDeclNames := make(map[string]bool) pruneOriginalFuncs := make(map[string]bool) @@ -307,53 +242,14 @@ func parseAndAugment(bctx *build.Context, pkg *build.Package, isTest bool, fileS importPath = importPath[:len(importPath)-5] } - nativesContext := &build.Context{ - GOROOT: "/", - GOOS: bctx.GOOS, - GOARCH: bctx.GOARCH, - Compiler: "gc", - JoinPath: path.Join, - SplitPathList: func(list string) []string { - if list == "" { - return nil - } - return strings.Split(list, "/") - }, - IsAbsPath: path.IsAbs, - IsDir: func(name string) bool { - dir, err := natives.FS.Open(name) - if err != nil { - return false - } - defer dir.Close() - info, err := dir.Stat() - if err != nil { - return false - } - return info.IsDir() - }, - HasSubdir: func(root, name string) (rel string, ok bool) { - panic("not implemented") - }, - ReadDir: func(name string) (fi []os.FileInfo, err error) { - dir, err := natives.FS.Open(name) - if err != nil { - return nil, err - } - defer dir.Close() - return dir.Readdir(0) - }, - OpenFile: func(name string) (r io.ReadCloser, err error) { - return natives.FS.Open(name) - }, - } + nativesContext := embeddedCtx(&withPrefix{fs: natives.FS, prefix: DefaultGOROOT}, "", nil) if importPath == "syscall" { // Special handling for the syscall package, which uses OS native // GOOS/GOARCH pair. This will no longer be necessary after // https://github.com/gopherjs/gopherjs/issues/693. - nativesContext.GOARCH = build.Default.GOARCH - nativesContext.BuildTags = append(nativesContext.BuildTags, "js") + nativesContext.bctx.GOARCH = build.Default.GOARCH + nativesContext.bctx.BuildTags = append(nativesContext.bctx.BuildTags, "js") } if nativesPkg, err := nativesContext.Import(importPath, "", 0); err == nil { @@ -366,7 +262,7 @@ func parseAndAugment(bctx *build.Context, pkg *build.Package, isTest bool, fileS } for _, name := range names { fullPath := path.Join(nativesPkg.Dir, name) - r, err := nativesContext.OpenFile(fullPath) + r, err := nativesContext.bctx.OpenFile(fullPath) if err != nil { panic(err) } @@ -406,7 +302,7 @@ func parseAndAugment(bctx *build.Context, pkg *build.Package, isTest bool, fileS if !filepath.IsAbs(name) { // name might be absolute if specified directly. E.g., `gopherjs build /abs/file.go`. name = filepath.Join(pkg.Dir, name) } - r, err := buildutil.OpenFile(bctx, name) + r, err := buildutil.OpenFile(pkg.bctx, name) if err != nil { return nil, err } @@ -509,6 +405,8 @@ func (o *Options) PrintSuccess(format string, a ...interface{}) { fmt.Fprintf(os.Stderr, format, a...) } +// PackageData is an extension of go/build.Package with additional metadata +// GopherJS requires. type PackageData struct { *build.Package JSFiles []string @@ -516,11 +414,50 @@ type PackageData struct { SrcModTime time.Time UpToDate bool IsVirtual bool // If true, the package does not have a corresponding physical directory on disk. + + bctx *build.Context // The original build context this package came from. +} + +// InternalBuildContext returns the build context that produced the package. +// +// WARNING: This function is a part of internal API and will be removed in +// future. +func (p *PackageData) InternalBuildContext() *build.Context { + return p.bctx +} + +// TestPackage returns a variant of the package with "internal" tests. +func (p *PackageData) TestPackage() *PackageData { + return &PackageData{ + Package: &build.Package{ + ImportPath: p.ImportPath, + Dir: p.Dir, + GoFiles: append(p.GoFiles, p.TestGoFiles...), + Imports: append(p.Imports, p.TestImports...), + }, + IsTest: true, + JSFiles: p.JSFiles, + bctx: p.bctx, + } +} + +// XTestPackage returns a variant of the package with "external" tests. +func (p *PackageData) XTestPackage() *PackageData { + return &PackageData{ + Package: &build.Package{ + ImportPath: p.ImportPath + "_test", + Dir: p.Dir, + GoFiles: p.XTestGoFiles, + Imports: p.XTestImports, + }, + IsTest: true, + bctx: p.bctx, + } } type Session struct { options *Options - bctx *build.Context + xctx XContext Archives map[string]*compiler.Archive Types map[string]*types.Package Watcher *fsnotify.Watcher @@ -544,7 +481,7 @@ func NewSession(options *Options) (*Session, error) { options: options, Archives: make(map[string]*compiler.Archive), } - s.bctx = NewBuildContext(s.InstallSuffix(), s.options.BuildTags) + s.xctx = NewBuildContext(s.InstallSuffix(), s.options.BuildTags) s.Types = make(map[string]*types.Package) if options.Watch { if out, err := exec.Command("ulimit", "-n").Output(); err == nil { @@ -563,7 +500,7 @@ func NewSession(options *Options) (*Session, error) { } // BuildContext returns the session's build context. -func (s *Session) BuildContext() *build.Context { return s.bctx } +func (s *Session) XContext() XContext { return s.xctx } func (s *Session) InstallSuffix() string { if s.options.Minify { @@ -576,16 +513,11 @@ func (s *Session) BuildDir(packagePath string, importPath string, pkgObj string) if s.Watcher != nil { s.Watcher.Add(packagePath) } - buildPkg, err := s.bctx.ImportDir(packagePath, 0) - if err != nil { - return err - } - pkg := &PackageData{Package: buildPkg} - jsFiles, err := jsFilesFromDir(s.bctx, pkg.Dir) + pkg, err := s.xctx.Import(".", packagePath, 0) if err != nil { return err } - pkg.JSFiles = jsFiles + archive, err := s.BuildPackage(pkg) if err != nil { return err @@ -608,6 +540,7 @@ func (s *Session) BuildFiles(filenames []string, pkgObj string, packagePath stri ImportPath: "main", Dir: packagePath, }, + bctx: &goCtx(s.InstallSuffix(), s.options.BuildTags).bctx, } for _, file := range filenames { @@ -634,7 +567,7 @@ func (s *Session) BuildImportPath(path string) (*compiler.Archive, error) { } func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*PackageData, *compiler.Archive, error) { - pkg, err := importWithSrcDir(*s.bctx, path, srcDir, 0, s.InstallSuffix()) + pkg, err := importWithSrcDir(s.xctx, path, srcDir, 0, s.InstallSuffix()) if s.Watcher != nil && pkg != nil { // add watch even on error s.Watcher.Add(pkg.Dir) } @@ -734,7 +667,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { } fileSet := token.NewFileSet() - files, err := parseAndAugment(s.bctx, pkg.Package, pkg.IsTest, fileSet) + files, err := parseAndAugment(s.xctx, pkg, pkg.IsTest, fileSet) if err != nil { return nil, err } diff --git a/build/build_test.go b/build/build_test.go index 659aff3e3..0eedacf05 100644 --- a/build/build_test.go +++ b/build/build_test.go @@ -8,7 +8,6 @@ import ( "strings" "testing" - "github.com/kisielk/gotool" "github.com/shurcooL/go/importgraphutil" ) @@ -64,7 +63,12 @@ func TestNativesDontImportExtraPackages(t *testing.T) { // Then, github.com/gopherjs/gopherjs/build.parseAndAugment(*build.Package) returns []*ast.File. // Those augmented parsed Go files of the package are checked, one file at at time, one import // at a time. Each import is verified to belong in the set of allowed real imports. - for _, pkg := range gotool.ImportPaths([]string{"std"}) { + matches, matchErr := simpleCtx{bctx: stdOnly}.Match([]string{"std"}) + if matchErr != nil { + t.Fatalf("Failed to list standard library packages: %s", err) + } + for _, pkg := range matches { + t.Logf("Checking package %s...", pkg) // Normal package. { // Import the real normal package, and populate its real import set. @@ -77,7 +81,7 @@ func TestNativesDontImportExtraPackages(t *testing.T) { // Use parseAndAugment to get a list of augmented AST files. fset := token.NewFileSet() - files, err := parseAndAugment(NewBuildContext("", nil), bpkg, false, fset) + files, err := parseAndAugment(NewBuildContext("", nil), &PackageData{Package: bpkg, bctx: &gobuild.Default}, false, fset) if err != nil { t.Fatalf("github.com/gopherjs/gopherjs/build.parseAndAugment: %v", err) } @@ -116,7 +120,7 @@ func TestNativesDontImportExtraPackages(t *testing.T) { // Use parseAndAugment to get a list of augmented AST files. fset := token.NewFileSet() - files, err := parseAndAugment(NewBuildContext("", nil), bpkg, true, fset) + files, err := parseAndAugment(NewBuildContext("", nil), &PackageData{Package: bpkg, bctx: &gobuild.Default}, true, fset) if err != nil { t.Fatalf("github.com/gopherjs/gopherjs/build.parseAndAugment: %v", err) } @@ -158,7 +162,7 @@ func TestNativesDontImportExtraPackages(t *testing.T) { // Use parseAndAugment to get a list of augmented AST files, then check only the external test files. fset := token.NewFileSet() - files, err := parseAndAugment(NewBuildContext("", nil), bpkg, true, fset) + files, err := parseAndAugment(NewBuildContext("", nil), &PackageData{Package: bpkg, bctx: &gobuild.Default}, true, fset) if err != nil { t.Fatalf("github.com/gopherjs/gopherjs/build.parseAndAugment: %v", err) } diff --git a/build/cache.go b/build/cache.go new file mode 100644 index 000000000..929afefe1 --- /dev/null +++ b/build/cache.go @@ -0,0 +1,35 @@ +package build + +import ( + "crypto/sha256" + "fmt" + "go/build" + "os" + "path/filepath" +) + +// cachePath is the base path for GopherJS's own build cache. +// +// It serves a similar function to the Go build cache, but is a lot more +// simlistic and therefore not compatible with Go. We use this cache directory +// to store build artifacts for packages loaded from a module, for which PkgObj +// provided by go/build points inside the module source tree, which can cause +// inconvenience with version control, etc. +var cachePath = func() string { + path, err := os.UserCacheDir() + if err == nil { + return filepath.Join(path, "gopherjs", "build_cache") + } + + return filepath.Join(build.Default.GOPATH, "pkg", "gopherjs_build_cache") +}() + +// cachedPath returns a location inside the build cache for a given PkgObj path +// returned by go/build. +func cachedPath(orig string) string { + if orig == "" { + panic(fmt.Errorf("CachedPath() must not be used with an empty string")) + } + sum := fmt.Sprintf("%x", sha256.Sum256([]byte(orig))) + return filepath.Join(cachePath, sum[0:2], sum) +} diff --git a/build/context.go b/build/context.go new file mode 100644 index 000000000..3a2279297 --- /dev/null +++ b/build/context.go @@ -0,0 +1,326 @@ +package build + +import ( + "fmt" + "go/build" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "sort" + "strings" + + "github.com/gopherjs/gopherjs/compiler" + "golang.org/x/tools/go/buildutil" +) + +// XContext is an extension of go/build.Context with GopherJS-specifc features. +// +// It abstracts away several different sources GopherJS can load its packages +// from, with a minimal API. +type XContext interface { + // Import returns details about the Go package named by the importPath, + // interpreting local import paths relative to the srcDir directory. + Import(path string, srcDir string, mode build.ImportMode) (*PackageData, error) + + // GOOS returns GOOS value the underlying build.Context is using. + // This will become obsolete after https://github.com/gopherjs/gopherjs/issues/693. + GOOS() string + + // Match explans build patterns into a set of matching import paths (see go help packages). + Match(patterns []string) ([]string, error) +} + +// simpleCtx is a wrapper around go/build.Context with support for GopherJS-specific +// features. +type simpleCtx struct { + bctx build.Context + isVirtual bool // Imported packages don't have a physical directory on disk. +} + +// Import implements XContext.Import(). +func (sc simpleCtx) Import(importPath string, srcDir string, mode build.ImportMode) (*PackageData, error) { + bctx, mode := sc.applyPackageTweaks(importPath, mode) + pkg, err := bctx.Import(importPath, srcDir, mode) + if err != nil { + return nil, err + } + jsFiles, err := jsFilesFromDir(&sc.bctx, pkg.Dir) + if err != nil { + return nil, fmt.Errorf("failed to enumerate .inc.js files in %s: %w", pkg.Dir, err) + } + pkg.PkgObj = sc.rewritePkgObj(pkg.PkgObj) + if !path.IsAbs(pkg.Dir) { + pkg.Dir = mustAbs(pkg.Dir) + } + return &PackageData{ + Package: pkg, + IsVirtual: sc.isVirtual, + JSFiles: jsFiles, + bctx: &sc.bctx, + }, nil +} + +// Match implements XContext.Match. +func (sc simpleCtx) Match(patterns []string) ([]string, error) { + if sc.isVirtual { + // We can't use go tool to enumerate packages in a virtual file system, + // so we fall back onto a simpler implementation provided by the buildutil + // package. It doesn't support all valid patterns, but should be good enough. + // + // Note: this code path will become unnecessary after + // https://github.com/gopherjs/gopherjs/issues/1021 is implemented. + args := []string{} + for _, p := range patterns { + switch p { + case "all": + args = append(args, "...") + case "std", "main", "cmd": + // These patterns are not supported by buildutil.ExpandPatterns(), + // but they would be matched by the real context correctly, so skip them. + default: + args = append(args, p) + } + } + matches := []string{} + for importPath := range buildutil.ExpandPatterns(&sc.bctx, args) { + if importPath[0] == '.' { + p, err := sc.Import(importPath, ".", build.FindOnly) + // Resolve relative patterns into canonical import paths. + if err != nil { + continue + } + importPath = p.ImportPath + } + matches = append(matches, importPath) + } + sort.Strings(matches) + return matches, nil + } + + args := append([]string{ + "-e", "-compiler=gc", + "-tags=" + strings.Join(sc.bctx.BuildTags, ","), + "-installsuffix=" + sc.bctx.InstallSuffix, + "-f={{.ImportPath}}", + "--", + }, patterns...) + + out, err := sc.gotool("list", args...) + if err != nil { + return nil, fmt.Errorf("failed to list packages on FS: %w", err) + } + matches := strings.Split(strings.TrimSpace(out), "\n") + sort.Strings(matches) + return matches, nil +} + +func (sc simpleCtx) GOOS() string { return sc.bctx.GOOS } + +// gotool executes the go tool set up for the build context and returns standard output. +func (sc simpleCtx) gotool(subcommand string, args ...string) (string, error) { + if sc.isVirtual { + panic(fmt.Errorf("can't use go tool with a virtual build context")) + } + args = append([]string{subcommand}, args...) + cmd := exec.Command("go", args...) + + if sc.bctx.Dir != "" { + cmd.Dir = sc.bctx.Dir + } + + var stdout, stderr strings.Builder + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + cgo := "0" + if sc.bctx.CgoEnabled { + cgo = "1" + } + cmd.Env = append(os.Environ(), + "GOOS="+sc.bctx.GOOS, + "GOARCH="+sc.bctx.GOARCH, + "GOROOT="+sc.bctx.GOROOT, + "GOPATH="+sc.bctx.GOPATH, + "CGO_ENABLED="+cgo, + ) + + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("go tool error: %v: %w\n%s", cmd, err, stderr.String()) + } + return stdout.String(), nil +} + +// applyPackageTweaks makes several package-specific adjustments to package importing. +// +// Ideally this method would not be necessary, but currently several packages +// require special handing in order to be compatible with GopherJS. This method +// returns a copy of the build context, keeping the original one intact. +func (sc simpleCtx) applyPackageTweaks(importPath string, mode build.ImportMode) (build.Context, build.ImportMode) { + bctx := sc.bctx + switch importPath { + case "syscall": + // syscall needs to use a typical GOARCH like amd64 to pick up definitions for _Socklen, BpfInsn, IFNAMSIZ, Timeval, BpfStat, SYS_FCNTL, Flock_t, etc. + bctx.GOARCH = build.Default.GOARCH + bctx.InstallSuffix += build.Default.GOARCH + case "syscall/js": + if !sc.isVirtual { + // There are no buildable files in this package upstream, but we need to use files in the virtual directory. + mode |= build.FindOnly + } + case "crypto/x509", "os/user": + // These stdlib packages have cgo and non-cgo versions (via build tags); we want the latter. + bctx.CgoEnabled = false + case "github.com/gopherjs/gopherjs/js", "github.com/gopherjs/gopherjs/nosync": + // These packages are already embedded via gopherjspkg.FS virtual filesystem (which can be + // safely vendored). Don't try to use vendor directory to resolve them. + mode |= build.IgnoreVendor + } + + return bctx, mode +} + +func (sc simpleCtx) rewritePkgObj(orig string) string { + if orig == "" { + return orig + } + + goroot := mustAbs(sc.bctx.GOROOT) + gopath := mustAbs(sc.bctx.GOPATH) + orig = mustAbs(orig) + + if strings.HasPrefix(orig, filepath.Join(gopath, "pkg", "mod")) { + // Go toolchain makes sources under GOPATH/pkg/mod readonly, so we can't + // store our artifacts there. + return cachedPath(orig) + } + + allowed := []string{goroot, gopath} + for _, prefix := range allowed { + if strings.HasPrefix(orig, prefix) { + // Traditional GOPATH-style locations for build artifacts are ok to use. + return orig + } + } + + // Everything else also goes into the cache just in case. + return cachedPath(orig) +} + +var defaultBuildTags = []string{ + "netgo", // See https://godoc.org/net#hdr-Name_Resolution. + "purego", // See https://golang.org/issues/23172. + "math_big_pure_go", // Use pure Go version of math/big. +} + +// embeddedCtx creates simpleCtx that imports from a virtual FS embedded into +// the GopherJS compiler. +func embeddedCtx(embedded http.FileSystem, installSuffix string, buildTags []string) *simpleCtx { + fs := &vfs{embedded} + ec := goCtx(installSuffix, buildTags) + ec.bctx.GOPATH = "" + + // Path functions must behave unix-like to work with the VFS. + ec.bctx.JoinPath = path.Join + ec.bctx.SplitPathList = splitPathList + ec.bctx.IsAbsPath = path.IsAbs + ec.bctx.HasSubdir = hasSubdir + + // Substitute real FS with the embedded one. + ec.bctx.IsDir = fs.IsDir + ec.bctx.ReadDir = fs.ReadDir + ec.bctx.OpenFile = fs.OpenFile + ec.isVirtual = true + return ec +} + +// goCtx creates simpleCtx that imports from the real file system GOROOT, GOPATH +// or Go Modules. +func goCtx(installSuffix string, buildTags []string) *simpleCtx { + gc := simpleCtx{ + bctx: build.Context{ + GOROOT: DefaultGOROOT, + GOPATH: build.Default.GOPATH, + GOOS: build.Default.GOOS, + GOARCH: "js", + InstallSuffix: installSuffix, + Compiler: "gc", + BuildTags: append(buildTags, defaultBuildTags...), + CgoEnabled: true, // detect `import "C"` to throw proper error + + // go/build supports modules, but only when no FS access functions are + // overridden and when provided ReleaseTags match those of the default + // context (matching Go compiler's version). + // This limitation stems from the fact that it will invoke the Go tool + // which can only see files on the real FS and will assume release tags + // based on the Go tool's version. + // + // See also comments to the versionhack package. + ReleaseTags: build.Default.ReleaseTags[:compiler.GoVersion], + }, + } + return &gc +} + +// chainedCtx combines two build contexts. Secondary context acts as a fallback +// when a package is not found in the primary, and is ignored otherwise. +// +// This allows GopherJS to load its core "js" and "nosync" packages from the +// embedded VFS whenever user's code doesn't directly depend on them, but +// augmented stdlib does. +type chainedCtx struct { + primary XContext + secondary XContext +} + +// Import implements buildCtx.Import(). +func (cc chainedCtx) Import(importPath string, srcDir string, mode build.ImportMode) (*PackageData, error) { + pkg, err := cc.primary.Import(importPath, srcDir, mode) + if err == nil { + return pkg, nil + } else if IsPkgNotFound(err) { + return cc.secondary.Import(importPath, srcDir, mode) + } else { + return nil, err + } +} + +func (cc chainedCtx) GOOS() string { return cc.primary.GOOS() } + +// Match implements XContext.Match(). +// +// Packages from both contexts are included and returned as a deduplicated +// sorted list. +func (cc chainedCtx) Match(patterns []string) ([]string, error) { + m1, err := cc.primary.Match(patterns) + if err != nil { + return nil, fmt.Errorf("failed to list packages in the primary context: %s", err) + } + m2, err := cc.secondary.Match(patterns) + if err != nil { + return nil, fmt.Errorf("failed to list packages in the secondary context: %s", err) + } + + seen := map[string]bool{} + matches := []string{} + for _, m := range append(m1, m2...) { + if seen[m] { + continue + } + seen[m] = true + matches = append(matches, m) + } + sort.Strings(matches) + return matches, nil +} + +// IsPkgNotFound returns true if the error was caused by package not found. +// +// Unfortunately, go/build doesn't make use of typed errors, so we have to +// rely on the error message. +func IsPkgNotFound(err error) bool { + return err != nil && + (strings.Contains(err.Error(), "cannot find package") || // Modules off. + strings.Contains(err.Error(), "is not in GOROOT")) // Modules on. +} diff --git a/build/context_test.go b/build/context_test.go new file mode 100644 index 000000000..844b2bc00 --- /dev/null +++ b/build/context_test.go @@ -0,0 +1,152 @@ +package build + +import ( + "fmt" + "go/build" + "path" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/gopherjs/gopherjs/compiler/gopherjspkg" + "golang.org/x/tools/go/buildutil" +) + +func TestSimpleCtx(t *testing.T) { + gopherjsRoot := filepath.Join(DefaultGOROOT, "src", "github.com", "gopherjs", "gopherjs") + fs := &withPrefix{gopherjspkg.FS, gopherjsRoot} + ec := embeddedCtx(fs, "", []string{}) + + gc := goCtx("", []string{}) + + t.Run("exists", func(t *testing.T) { + tests := []struct { + buildCtx XContext + wantPkg *PackageData + }{ + { + buildCtx: ec, + wantPkg: &PackageData{ + Package: expectedPackage(&ec.bctx, "github.com/gopherjs/gopherjs/js"), + IsVirtual: true, + }, + }, { + buildCtx: gc, + wantPkg: &PackageData{ + Package: expectedPackage(&gc.bctx, "fmt"), + IsVirtual: false, + }, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%T", test.buildCtx), func(t *testing.T) { + importPath := test.wantPkg.ImportPath + got, err := test.buildCtx.Import(importPath, "", build.FindOnly) + if err != nil { + t.Fatalf("ec.Import(%q) returned error: %s. Want: no error.", importPath, err) + } + if diff := cmp.Diff(test.wantPkg, got, cmpopts.IgnoreUnexported(*got)); diff != "" { + t.Errorf("ec.Import(%q) returned diff (-want,+got):\n%s", importPath, diff) + } + }) + } + }) + + t.Run("not found", func(t *testing.T) { + tests := []struct { + buildCtx XContext + importPath string + }{ + { + buildCtx: ec, + importPath: "package/not/found", + }, { + // Outside of the main module. + buildCtx: gc, + importPath: "package/not/found", + }, { + // In the main module. + buildCtx: gc, + importPath: "github.com/gopherjs/gopherjs/not/found", + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%T", test.buildCtx), func(t *testing.T) { + _, err := ec.Import(test.importPath, "", build.FindOnly) + want := "cannot find package" + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("ec.Import(%q) returned error: %s. Want error containing %q.", test.importPath, err, want) + } + }) + } + }) +} + +func TestChainedCtx(t *testing.T) { + // Construct a chained context of two fake contexts so that we could verify + // fallback behavior. + cc := chainedCtx{ + primary: simpleCtx{ + bctx: *buildutil.FakeContext(map[string]map[string]string{ + "primaryonly": {"po.go": "package primaryonly"}, + "both": {"both.go": "package both"}, + }), + isVirtual: false, + }, + secondary: simpleCtx{ + bctx: *buildutil.FakeContext(map[string]map[string]string{ + "both": {"both_secondary.go": "package both"}, + "secondaryonly": {"so.go": "package secondaryonly"}, + }), + isVirtual: true, + }, + } + + tests := []struct { + importPath string + wantFromPrimary bool + }{ + { + importPath: "primaryonly", + wantFromPrimary: true, + }, { + importPath: "both", + wantFromPrimary: true, + }, { + importPath: "secondaryonly", + wantFromPrimary: false, + }, + } + + for _, test := range tests { + t.Run(test.importPath, func(t *testing.T) { + pkg, err := cc.Import(test.importPath, "", 0) + if err != nil { + t.Errorf("cc.Import() returned error: %v. Want: no error.", err) + } + gotFromPrimary := !pkg.IsVirtual + if gotFromPrimary != test.wantFromPrimary { + t.Errorf("Got package imported from primary: %t. Want: %t.", gotFromPrimary, test.wantFromPrimary) + } + }) + } +} + +func expectedPackage(bctx *build.Context, importPath string) *build.Package { + targetRoot := path.Clean(fmt.Sprintf("%s/pkg/%s_%s", bctx.GOROOT, bctx.GOOS, bctx.GOARCH)) + return &build.Package{ + Dir: path.Join(bctx.GOROOT, "src", importPath), + ImportPath: importPath, + Root: bctx.GOROOT, + SrcRoot: path.Join(bctx.GOROOT, "src"), + PkgRoot: path.Join(bctx.GOROOT, "pkg"), + PkgTargetRoot: targetRoot, + BinDir: path.Join(bctx.GOROOT, "bin"), + Goroot: true, + PkgObj: path.Join(targetRoot, importPath+".a"), + } +} diff --git a/build/fsutil.go b/build/fsutil.go new file mode 100644 index 000000000..cee763f14 --- /dev/null +++ b/build/fsutil.go @@ -0,0 +1,28 @@ +package build + +import ( + "fmt" + "os" + "path/filepath" +) + +func mustAbs(p string) string { + a, err := filepath.Abs(p) + if err != nil { + panic(fmt.Errorf("failed to get absolute path to %s", p)) + } + return a +} + +// makeWritable attempts to make the given path writable by its owner. +func makeWritable(path string) error { + info, err := os.Stat(path) + if err != nil { + return err + } + err = os.Chmod(path, info.Mode()|0700) + if err != nil { + return err + } + return nil +} diff --git a/build/versionhack/versionhack.go b/build/versionhack/versionhack.go new file mode 100644 index 000000000..1d7f1b68b --- /dev/null +++ b/build/versionhack/versionhack.go @@ -0,0 +1,42 @@ +// Package versionhack makes sure go/build doesn't disable module support +// whenever GopherJS is compiled by a different Go version than it's targeted +// Go version. +// +// Under the hood, go/build relies on `go list` utility for module support; more +// specifically, for package location discovery. Since ReleaseTags are +// effectively baked into the go binary and can't be overridden, it needs to +// ensure that ReleaseTags set in a go/build.Context instance match the Go tool. +// +// However, it naively assumes that the go tool version in the PATH matches the +// version that was used to build GopherJS and disables module support whenever +// ReleaseTags in the context are set to anything other than the default. This, +// unfortunately, isn't very helpful since gopherjs may be build by a Go version +// other than the PATH's default. +// +// Luckily, even if go tool version is mismatches, it's only used for discovery +// of the package locations, and go/build evaluates build constraints on its own +// with ReleaseTags we've passed. +// +// A better solution would've been for go/build to use go tool from GOROOT and +// check its version against build tags: https://github.com/golang/go/issues/46856. +// +// Until that issue is fixed, we trick go/build into thinking that whatever +// ReleaseTags we've passed are indeed the default. We gain access to the +// variable go/build checks against using "go:linkname" directive and override +// its content as we wish. +package versionhack + +import ( + "go/build" // Must be initialized before this package. + + "github.com/gopherjs/gopherjs/compiler" + + _ "unsafe" // For go:linkname +) + +//go:linkname releaseTags go/build.defaultReleaseTags +var releaseTags []string + +func init() { + releaseTags = build.Default.ReleaseTags[:compiler.GoVersion] +} diff --git a/build/vfs.go b/build/vfs.go new file mode 100644 index 000000000..ffc041c4f --- /dev/null +++ b/build/vfs.go @@ -0,0 +1,86 @@ +package build + +import ( + "io" + "net/http" + "os" + "path" + "strings" +) + +// vfs is a convenience wrapper around http.FileSystem that provides accessor +// methods required by go/build.Context. +type vfs struct{ http.FileSystem } + +func (fs vfs) IsDir(name string) bool { + dir, err := fs.Open(name) + if err != nil { + return false + } + defer dir.Close() + info, err := dir.Stat() + if err != nil { + return false + } + return info.IsDir() +} + +func (fs vfs) ReadDir(name string) (fi []os.FileInfo, err error) { + dir, err := fs.Open(name) + if err != nil { + return nil, err + } + defer dir.Close() + return dir.Readdir(0) +} + +func (fs vfs) OpenFile(name string) (r io.ReadCloser, err error) { + return fs.Open(name) +} + +func splitPathList(list string) []string { + if list == "" { + return nil + } + const pathListSeparator = ":" // UNIX style + return strings.Split(list, pathListSeparator) +} + +// hasSubdir reports whether dir is lexically a subdirectory of +// root, perhaps multiple levels below. It does not try to check +// whether dir exists. +// If so, hasSubdir sets rel to a slash-separated path that +// can be joined to root to produce a path equivalent to dir. +func hasSubdir(root, dir string) (rel string, ok bool) { + // Implementation based on golang.org/x/tools/go/buildutil. + const sep = "/" // UNIX style + root = path.Clean(root) + if !strings.HasSuffix(root, sep) { + root += sep + } + + dir = path.Clean(dir) + if !strings.HasPrefix(dir, root) { + return "", false + } + + return dir[len(root):], true +} + +// withPrefix implements http.FileSystem, which places the underlying FS under +// the given prefix path. +type withPrefix struct { + fs http.FileSystem + prefix string +} + +func (wp *withPrefix) Open(name string) (http.File, error) { + if !strings.HasPrefix(name, wp.prefix) { + return nil, &os.PathError{Op: "open", Path: name, Err: os.ErrNotExist} + } + f, err := wp.fs.Open(strings.TrimPrefix(name, wp.prefix)) + if err != nil { + return nil, &os.PathError{Op: "open", Path: name, Err: err} + } + return f, nil +} diff --git a/circle.yml b/circle.yml index dccdf25de..b7e5924c9 100644 --- a/circle.yml +++ b/circle.yml @@ -5,8 +5,8 @@ jobs: - image: ubuntu:18.04 environment: SOURCE_MAP_SUPPORT: true - GO111MODULE: "off" # Until issue #855 is fixed, we operate in GOPATH mode. - working_directory: ~/go/src/github.com/gopherjs/gopherjs + GO111MODULE: "on" + working_directory: ~/gopherjs steps: - run: apt-get update && apt-get install -y sudo curl git python make g++ - checkout @@ -29,6 +29,31 @@ jobs: - run: for d in */; do echo ./$d...; done | grep -v ./doc | grep -v ./tests | grep -v ./node | xargs go vet # All subdirectories except "doc", "tests", "node*". - run: diff -u <(echo -n) <(go list ./compiler/natives/src/...) # All those packages should have // +build js. - run: gopherjs install -v net/http # Should build successfully (can't run tests, since only client is supported). - - run: ulimit -s 10000 && gopherjs test --minify -v --short github.com/gopherjs/gopherjs/js/... github.com/gopherjs/gopherjs/tests/... $(go list std | grep -v -x -f .std_test_pkg_exclusions) + - run: ulimit -s 10000 && gopherjs test --minify -v --short github.com/gopherjs/gopherjs/js/... github.com/gopherjs/gopherjs/tests/... $(GOOS=js GOARCH=wasm go list std | grep -v -x -f .std_test_pkg_exclusions) - run: go test -v -race ./... - run: gopherjs test -v fmt # No minification should work. + - run: + name: TodoMVC in GOPATH mode + command: | + export GO111MODULE=off + export GOPATH=/tmp/gopath + mkdir -p $GOPATH/src + go get -v github.com/gopherjs/todomvc + gopherjs build -v -o /tmp/todomvc_gopath.js github.com/gopherjs/todomvc + gopherjs test -v github.com/gopherjs/todomvc/... + find $GOPATH + - run: + name: TodoMVC in Go Modules mode + command: | + export GO111MODULE=on + export GOPATH=/tmp/gomod + mkdir -p $GOPATH/src + cd /tmp + git clone --depth=1 https://github.com/gopherjs/todomvc.git + cd /tmp/todomvc + gopherjs build -v -o /tmp/todomvc_gomod.js github.com/gopherjs/todomvc + gopherjs test -v github.com/gopherjs/todomvc/... + find $GOPATH + - run: + name: Compare GOPATH and Go Modules output + command: diff -u <(sed 's/todomvc_gomod.js.map/todomvc_ignored.js.map/' /tmp/todomvc_gomod.js) <(sed 's/todomvc_gopath.js.map/todomvc_ignored.js.map/' /tmp/todomvc_gopath.js) diff --git a/go.mod b/go.mod index 9d9b50bba..de01b08f8 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.16 require ( github.com/fsnotify/fsnotify v1.4.9 github.com/google/go-cmp v0.5.5 - github.com/kisielk/gotool v1.0.0 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86 github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636 diff --git a/go.sum b/go.sum index 6803750ed..77178cd09 100644 --- a/go.sum +++ b/go.sum @@ -100,7 +100,6 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= diff --git a/tests/gopherjsvendored_test.sh b/tests/gopherjsvendored_test.sh index a9ef4105f..e7b8097ea 100755 --- a/tests/gopherjsvendored_test.sh +++ b/tests/gopherjsvendored_test.sh @@ -40,6 +40,7 @@ done # Make $tmp our GOPATH workspace. export GOPATH="$tmp" +export GO111MODULE=off # Build the vendored copy of GopherJS. go install example.org/hello/vendor/github.com/gopherjs/gopherjs diff --git a/tests/misc_test.go b/tests/misc_test.go index 2f27d6983..9d1577b08 100644 --- a/tests/misc_test.go +++ b/tests/misc_test.go @@ -7,7 +7,6 @@ import ( "strings" "testing" "time" - "vendored" "github.com/gopherjs/gopherjs/tests/otherpkg" ) @@ -504,12 +503,6 @@ func TestGoexit(t *testing.T) { }() } -func TestVendoring(t *testing.T) { - if vendored.Answer != 42 { - t.Fail() - } -} - func TestShift(t *testing.T) { if x := uint(32); uint32(1)< 1 { + if *compileOnly && len(matches) > 1 { return errors.New("cannot use -c flag with multiple packages") } - if *outputFilename != "" && len(args) > 1 { + if *outputFilename != "" && len(matches) > 1 { return errors.New("cannot use -o flag with multiple packages") } - pkgs := make([]*gbuild.PackageData, len(args)) - for i, pkgPath := range args { + pkgs := make([]*gbuild.PackageData, len(matches)) + for i, pkgPath := range matches { var err error pkgs[i], err = gbuild.Import(pkgPath, 0, "", options.BuildTags) if err != nil { @@ -346,7 +355,7 @@ func main() { return err } - tests := &testFuncs{BuildContext: s.BuildContext(), Package: pkg.Package} + tests := &testFuncs{BuildContext: pkg.InternalBuildContext(), Package: pkg.Package} collectTests := func(testPkg *gbuild.PackageData, testPkgName string, needVar *bool) error { if testPkgName == "_test" { for _, file := range pkg.TestGoFiles { @@ -365,28 +374,11 @@ func main() { return err } - if err := collectTests(&gbuild.PackageData{ - Package: &build.Package{ - ImportPath: pkg.ImportPath, - Dir: pkg.Dir, - GoFiles: append(pkg.GoFiles, pkg.TestGoFiles...), - Imports: append(pkg.Imports, pkg.TestImports...), - }, - IsTest: true, - JSFiles: pkg.JSFiles, - }, "_test", &tests.NeedTest); err != nil { + if err := collectTests(pkg.TestPackage(), "_test", &tests.NeedTest); err != nil { return err } - if err := collectTests(&gbuild.PackageData{ - Package: &build.Package{ - ImportPath: pkg.ImportPath + "_test", - Dir: pkg.Dir, - GoFiles: pkg.XTestGoFiles, - Imports: pkg.XTestImports, - }, - IsTest: true, - }, "_xtest", &tests.NeedXtest); err != nil { + if err := collectTests(pkg.XTestPackage(), "_xtest", &tests.NeedXtest); err != nil { return err } @@ -495,7 +487,6 @@ func main() { cmdServe.Flags().StringVarP(&addr, "http", "", ":8080", "HTTP bind address to serve") cmdServe.Run = func(cmd *cobra.Command, args []string) { options.BuildTags = strings.Fields(tags) - dirs := append(filepath.SplitList(build.Default.GOPATH), gbuild.DefaultGOROOT) var root string if len(args) > 1 { @@ -517,7 +508,6 @@ func main() { sourceFiles := http.FileServer(serveCommandFileSystem{ serveRoot: root, options: options, - dirs: dirs, sourceMaps: make(map[string][]byte), }) @@ -579,12 +569,12 @@ func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) { type serveCommandFileSystem struct { serveRoot string options *gbuild.Options - dirs []string sourceMaps map[string][]byte } func (fs serveCommandFileSystem) Open(requestName string) (http.File, error) { name := path.Join(fs.serveRoot, requestName[1:]) // requestName[0] == '/' + log.Printf("Request: %s", name) dir, file := path.Split(name) base := path.Base(dir) // base is parent folder name, which becomes the output file name. @@ -593,13 +583,14 @@ func (fs serveCommandFileSystem) Open(requestName string) (http.File, error) { isMap := file == base+".js.map" isIndex := file == "index.html" + // Create a new session to pick up changes to source code on disk. + // TODO(dmitshur): might be possible to get a single session to detect changes to source code on disk + s, err := gbuild.NewSession(fs.options) + if err != nil { + return nil, err + } + if isPkg || isMap || isIndex { - // Create a new session to pick up changes to source code on disk. - // TODO(dmitshur): might be possible to get a single session to detect changes to source code on disk - s, err := gbuild.NewSession(fs.options) - if err != nil { - return nil, err - } // If we're going to be serving our special files, make sure there's a Go command in this folder. pkg, err := gbuild.Import(path.Dir(name), 0, s.InstallSuffix(), fs.options.BuildTags) if err != nil || pkg.Name != "main" { @@ -650,19 +641,14 @@ func (fs serveCommandFileSystem) Open(requestName string) (http.File, error) { } } - for _, d := range fs.dirs { - dir := http.Dir(filepath.Join(d, "src")) - - f, err := dir.Open(name) - if err == nil { - return f, nil - } + // First try to serve the request with a root prefix supplied in the CLI. + if f, err := fs.serveSourceTree(s.XContext(), name); err == nil { + return f, nil + } - // source maps are served outside of serveRoot - f, err = dir.Open(requestName) - if err == nil { - return f, nil - } + // If that didn't work, try without the prefix. + if f, err := fs.serveSourceTree(s.XContext(), requestName); err == nil { + return f, nil } if isIndex { @@ -673,6 +659,24 @@ func (fs serveCommandFileSystem) Open(requestName string) (http.File, error) { return nil, os.ErrNotExist } +func (fs serveCommandFileSystem) serveSourceTree(xctx gbuild.XContext, reqPath string) (http.File, error) { + parts := strings.Split(path.Clean(reqPath), "/") + // Under Go Modules different packages can be located in different module + // directories, which no longer align with import paths. + // + // We don't know which part of the requested path is package import path and + // which is a path under the package directory, so we try different slipt + // points until the package is found successfully. + for i := len(parts); i > 0; i-- { + pkgPath := path.Clean(path.Join(parts[:i]...)) + filePath := path.Clean(path.Join(parts[i:]...)) + if pkg, err := xctx.Import(pkgPath, ".", build.FindOnly); err == nil { + return http.Dir(pkg.Dir).Open(filePath) + } + } + return nil, os.ErrNotExist +} + type fakeFile struct { name string size int