From 587ec90ef2ed1378ebfe602a8cc10bc657871191 Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Sun, 6 Sep 2015 22:36:16 -0700 Subject: [PATCH] Add Go repository compiler tests, triage known failures. Add a modified version of go/test/run.go for GopherJS. The run.go file should be run via native Go, but it will itself invoke `gopherjs run` for running tests. All modifications are marked with a "// GOPHERJS" comment. At this time, only a limited subset of folders and test types is supported. More can be added. Within the tests that are run, there are currently 234 tests passing and 34 failing. The failing test are marked as known failures and do not trigger CI failure. As the bugs are resolved, the knownFails map in run.go should be updated to remove resolved failures. The test will currently fail if any of the tests that were expected to fail unexpectedly pass (if it's not an accident, then knownFails map should be updated accordingly). I hope this will not cause too much noise; it can be changed if so. Four of the tests are compiler panics, they should probably be fixed first. The remaining ones are briefly described but not triaged further than that. The run.go test runner can be run manually via: go run run.go -v -summary Or to include known fails full error messages: go run run.go -v -summary -show_known_fails Add fixedbugs/issue10607.go to known failures. It cannot run because it uses os/exec. This test is meaningless anyway because it tests native go compiler linking modes. Remove unneeded TODOs. Don't output details of known test failures unless show_known_fails flag is on. This makes normal verbose output much more readable and suitable for CI. Change TestGoRepositoryCompilerTests to stream output as it runs. Previously, it would only display all of output when it finished. Fix issue where specifying specific test to run failed to run it. It was because filepath.Split left a trailing slash after dir, so "fixedbugs/" != "fixedbugs". Use filepath.Clean on input t.dir to resolve this. Always display known fails details if -show_known_fails flag is on (even without -v). Previously, both -v and -show_known_fails had to be on. Also tweak the reproduce command printed to include -show_known_fails flag, since it's needed. Remove absolute paths from known fails descriptions. --- tests/gorepo_test.go | 24 + tests/run.go | 1209 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1233 insertions(+) create mode 100644 tests/gorepo_test.go create mode 100644 tests/run.go diff --git a/tests/gorepo_test.go b/tests/gorepo_test.go new file mode 100644 index 000000000..5496cd32e --- /dev/null +++ b/tests/gorepo_test.go @@ -0,0 +1,24 @@ +// +build !js + +package tests_test + +import ( + "os" + "os/exec" + "testing" +) + +// Go repository basic compiler tests, and regression tests for fixed compiler bugs. +func TestGoRepositoryCompilerTests(t *testing.T) { + args := []string{"go", "run", "run.go", "-summary"} + if testing.Verbose() { + args = append(args, "-v") + } + cmd := exec.Command(args[0], args[1:]...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stdout + err := cmd.Run() + if err != nil { + t.Fatal(err) + } +} diff --git a/tests/run.go b/tests/run.go new file mode 100644 index 000000000..dfbfc97ab --- /dev/null +++ b/tests/run.go @@ -0,0 +1,1209 @@ +// +build ignore + +// skip + +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Run runs tests in the test directory. +// +// To run manually with summary, verbose output, and full stack traces of of known failures: +// +// go run run.go -summary -v -show_known_fails +// +// TODO(bradfitz): docs of some sort, once we figure out how we're changing +// headers of files +package main + +import ( + "bytes" + "errors" + "flag" + "fmt" + "hash/fnv" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "runtime" + "sort" + "strconv" + "strings" + "time" + "unicode" +) + +// ----------------------------------------------------------------------------- +// GOPHERJS: Known test fails for GopherJS compiler. +// +// TODO: Reduce these to zero or as close as possible. +// +var knownFails = map[string]failReason{ + // GopherJS compiler panics. + "fixedbugs/bug180.go": {category: compilerPanic, desc: ` +panic: 20 + +goroutine 1 [running]: +github.com/gopherjs/gopherjs/compiler.(*funcContext).fixNumber(0xc820164210, 0xc82000f360, 0x9088a0, 0xc820018ea0) + github.com/gopherjs/gopherjs/compiler/expressions.go:1176 +0x235 +`}, + "fixedbugs/issue12133.go": {category: compilerPanic, desc: ` +github.com/gopherjs/gopherjs/compiler.(*funcContext).fixNumber(0xc82027c840, 0xc82004c820, 0x9088a0, 0xc82004c6c0) + github.com/gopherjs/gopherjs/compiler/expressions.go:1176 +0x235 +`}, + "fixedbugs/issue5793.go": {category: compilerPanic, desc: ` +panic: runtime error: index out of range + +goroutine 1 [running]: +github.com/gopherjs/gopherjs/compiler.(*funcContext).translateBuiltin(0xc820160420, 0x6331c0, 0x7, 0xc8201344e0, 0x1, 0x1, 0xc820135900, 0xe04788, 0x908820, 0xc820135910) + github.com/gopherjs/gopherjs/compiler/expressions.go:882 +0x41c7 +`}, + "fixedbugs/issue9691.go": {category: compilerPanic, desc: ` +panic: Unhandled lhs type: *types.Map + +goroutine 1 [running]: +github.com/gopherjs/gopherjs/compiler.(*funcContext).translateAssign(0xc820160210, 0xe00488, 0xc820105050, 0x632508, 0x2, 0xcb46b0, 0x908660, 0x0, 0x0, 0x0) + github.com/gopherjs/gopherjs/compiler/statements.go:785 +0x1931 +`}, + + "fixedbugs/bug243.go": {desc: "TypeError: undefined is not a function"}, + "fixedbugs/issue4495.go": {desc: "TypeError: Cannot read property 'm' of undefined"}, + + "fixedbugs/bug114.go": {desc: "fixedbugs/bug114.go:15:27: B32 (untyped int constant 4294967295) overflows int"}, + "fixedbugs/bug242.go": {desc: "bad map check 13 false false Error: fail"}, + "fixedbugs/bug260.go": {desc: "maybe unsupportedFeature, pointer arithm"}, + "fixedbugs/bug262.go": {desc: "Error: fail"}, + "fixedbugs/bug273.go": {desc: "BUG: didn't crash: badcap1"}, + "fixedbugs/bug328.go": {desc: "incorrect output"}, + "fixedbugs/bug347.go": {desc: "BUG: bug347: cannot find caller"}, + "fixedbugs/bug348.go": {desc: "BUG: bug348: cannot find caller"}, + "fixedbugs/bug352.go": {desc: "BUG: bug352 struct{}"}, + "fixedbugs/bug356.go": {desc: "BUG bug344 0"}, + "fixedbugs/bug409.go": {desc: "1 2 3 4"}, + "fixedbugs/bug433.go": {desc: "Error: [object Object]"}, + "fixedbugs/bug461.go": {desc: `SyntaxError: Unexpected token . + var $pkg = {}, $init, reflect, structType, $pkg.T$ptr, main; + ^`}, + "fixedbugs/bug491.go": {desc: "BUG: append call not ordered: 3 2 3"}, + "fixedbugs/issue10353.go": {desc: "incorrect output"}, + "fixedbugs/issue11256.go": {desc: "null"}, + "fixedbugs/issue11656.go": {desc: "Error: Native function not implemented: runtime/debug.setPanicOnFault"}, + "fixedbugs/issue4085b.go": {desc: "Error: got panic JavaScript error: Invalid typed array length, want len out of range"}, + "fixedbugs/issue4316.go": {desc: "Error: runtime error: invalid memory address or nil pointer dereference"}, + "fixedbugs/issue4388.go": {desc: "Error: expected :1 have anonymous function:0"}, + "fixedbugs/issue4562.go": {desc: "Error: cannot find issue4562.go on stack"}, + "fixedbugs/issue4620.go": {desc: "map[0:1 1:2], Error: m[i] != 2"}, + "fixedbugs/issue5856.go": {desc: "BUG: defer called from /private/var/folders/tw/kgz4v2kn4n7d7ryg5k_z3dk40000gn/T/issue5856.go.931062983:15135, want issue5856.go:28"}, + "fixedbugs/issue6899.go": {desc: "incorrect output -0"}, + "fixedbugs/issue7550.go": {desc: "FATAL ERROR: invalid table size Allocation failed - process out of memory"}, + "fixedbugs/issue7690.go": {desc: "Error: runtime error: slice bounds out of range"}, + "fixedbugs/issue8047.go": {desc: "null"}, + "fixedbugs/issue8047b.go": {desc: "Error: [object Object]"}, + + // Failing due to use of os/exec.Command, which is unsupported. Now skipped via !nacl build tag. + /*"fixedbugs/bug248.go": {desc: "Error: Native function not implemented: syscall.runtime_BeforeFork"}, + "fixedbugs/bug302.go": {desc: "Error: Native function not implemented: syscall.runtime_BeforeFork"}, + "fixedbugs/bug345.go": {desc: "Error: Native function not implemented: syscall.runtime_BeforeFork"}, + "fixedbugs/bug369.go": {desc: "Error: Native function not implemented: syscall.runtime_BeforeFork"}, + "fixedbugs/bug429_run.go": {desc: "Error: Native function not implemented: syscall.runtime_BeforeFork"}, + "fixedbugs/issue9862_run.go": {desc: "Error: Native function not implemented: syscall.runtime_BeforeFork"},*/ + "fixedbugs/issue10607.go": {desc: "Error: Native function not implemented: syscall.runtime_BeforeFork"}, +} + +type failCategory uint8 + +const ( + other failCategory = iota + compilerPanic +) + +type failReason struct { + category failCategory + desc string +} + +// ----------------------------------------------------------------------------- + +var ( + verbose = flag.Bool("v", false, "verbose. if set, parallelism is set to 1.") + numParallel = flag.Int("n", runtime.NumCPU(), "number of parallel tests to run") + summary = flag.Bool("summary", false, "show summary of results") + showSkips = flag.Bool("show_skips", false, "show skipped tests") + showKnownFails = flag.Bool("show_known_fails", false, "show full error output of known fails") + updateErrors = flag.Bool("update_errors", false, "update error messages in test file based on compiler output") + runoutputLimit = flag.Int("l", defaultRunOutputLimit(), "number of parallel runoutput tests to run") + + shard = flag.Int("shard", 0, "shard index to run. Only applicable if -shards is non-zero.") + shards = flag.Int("shards", 0, "number of shards. If 0, all tests are run. This is used by the continuous build.") +) + +var ( + goos, goarch string + + // dirs are the directories to look for *.go files in. + // TODO(bradfitz): just use all directories? + dirs = []string{".", "ken", "chan", "interface", "syntax", "dwarf", "fixedbugs", "bugs"} + + // ratec controls the max number of tests running at a time. + ratec chan bool + + // toRun is the channel of tests to run. + // It is nil until the first test is started. + toRun chan *test + + // rungatec controls the max number of runoutput tests + // executed in parallel as they can each consume a lot of memory. + rungatec chan bool +) + +// maxTests is an upper bound on the total number of tests. +// It is used as a channel buffer size to make sure sends don't block. +const maxTests = 5000 + +func main() { + flag.Parse() + + // GOPHERJS. + err := os.Chdir(filepath.Join(runtime.GOROOT(), "test")) + if err != nil { + log.Fatalln(err) + } + + goos = getenv("GOOS", runtime.GOOS) + //goarch = getenv("GOARCH", runtime.GOARCH) + // GOPHERJS. + goarch = getenv("GOARCH", "js") // We're running this script natively, but the tests are executed with js architecture. + + if *verbose { + fmt.Printf("goos: %q, goarch: %q\n", goos, goarch) + } + + findExecCmd() + + // Disable parallelism if printing or if using a simulator. + if *verbose || len(findExecCmd()) > 0 { + *numParallel = 1 + } + + ratec = make(chan bool, *numParallel) + rungatec = make(chan bool, *runoutputLimit) + + var tests []*test + if flag.NArg() > 0 { + for _, arg := range flag.Args() { + if arg == "-" || arg == "--" { + // Permit running: + // $ go run run.go - env.go + // $ go run run.go -- env.go + // $ go run run.go - ./fixedbugs + // $ go run run.go -- ./fixedbugs + continue + } + if fi, err := os.Stat(arg); err == nil && fi.IsDir() { + for _, baseGoFile := range goFiles(arg) { + tests = append(tests, startTest(arg, baseGoFile)) + } + } else if strings.HasSuffix(arg, ".go") { + dir, file := filepath.Split(arg) + tests = append(tests, startTest(dir, file)) + } else { + log.Fatalf("can't yet deal with non-directory and non-go file %q", arg) + } + } + } else { + for _, dir := range dirs { + for _, baseGoFile := range goFiles(dir) { + tests = append(tests, startTest(dir, baseGoFile)) + } + } + } + + failed := false + resCount := map[string]int{} + for _, test := range tests { + <-test.donec + // GOPHERJS. + if test.action == "skip" && !*showSkips { + continue + } + status := "ok " + errStr := "" + if _, isSkip := test.err.(skipError); isSkip { + test.err = nil + errStr = "unexpected skip for " + path.Join(test.dir, test.gofile) + ": " + errStr + status = "FAIL" + } + // GOPHERJS. + if _, ok := knownFails[filepath.ToSlash(test.goFileName())]; ok && test.err != nil { + errStr = test.err.Error() + test.err = nil + status = "knfl" // knfl means known failure. Expect test to fail. + } else if ok && test.err == nil { + // unok means unexpected okay. Test was expected to fail, but it unexpectedly succeeded. + // If this is not an accident, it should be removed from knownFails map. + status = "unok" + } + if test.err != nil { + status = "FAIL" + errStr = test.err.Error() + } + if status == "FAIL" { + failed = true + } + // GOPHERJS. + if status == "unok" { + failed = true + } + resCount[status]++ + if status == "skip" && !*verbose && !*showSkips { + continue + } + dt := fmt.Sprintf("%.3fs", test.dt.Seconds()) + if status == "FAIL" { + fmt.Printf("# go run run.go -- %s\n%s\nFAIL\t%s\t%s\n", + path.Join(test.dir, test.gofile), + errStr, test.goFileName(), dt) + continue + } + // GOPHERJS. + if status == "knfl" && *showKnownFails { + fmt.Printf("# go run run.go -show_known_fails -- %s\n%s\nknfl\t%s\t%s\n", + path.Join(test.dir, test.gofile), + errStr, test.goFileName(), dt) + continue + } + if !*verbose { + continue + } + fmt.Printf("%s\t%s\t%s\n", status, test.goFileName(), dt) + } + + if *summary { + for k, v := range resCount { + fmt.Printf("%5d %s\n", v, k) + } + } + + if failed { + os.Exit(1) + } +} + +func toolPath(name string) string { + p := filepath.Join(os.Getenv("GOROOT"), "bin", "tool", name) + if _, err := os.Stat(p); err != nil { + log.Fatalf("didn't find binary at %s", p) + } + return p +} + +func shardMatch(name string) bool { + if *shards == 0 { + return true + } + h := fnv.New32() + io.WriteString(h, name) + return int(h.Sum32()%uint32(*shards)) == *shard +} + +func goFiles(dir string) []string { + f, err := os.Open(dir) + check(err) + dirnames, err := f.Readdirnames(-1) + check(err) + names := []string{} + for _, name := range dirnames { + if !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".go") && shardMatch(name) { + names = append(names, name) + } + } + sort.Strings(names) + return names +} + +type runCmd func(...string) ([]byte, error) + +func compileFile(runcmd runCmd, longname string) (out []byte, err error) { + return runcmd("go", "tool", "compile", "-e", longname) +} + +func compileInDir(runcmd runCmd, dir string, names ...string) (out []byte, err error) { + cmd := []string{"go", "tool", "compile", "-e", "-D", ".", "-I", "."} + for _, name := range names { + cmd = append(cmd, filepath.Join(dir, name)) + } + return runcmd(cmd...) +} + +func linkFile(runcmd runCmd, goname string) (err error) { + pfile := strings.Replace(goname, ".go", ".o", -1) + _, err = runcmd("go", "tool", "link", "-w", "-o", "a.exe", "-L", ".", pfile) + return +} + +// skipError describes why a test was skipped. +type skipError string + +func (s skipError) Error() string { return string(s) } + +func check(err error) { + if err != nil { + log.Fatal(err) + } +} + +// test holds the state of a test. +type test struct { + dir, gofile string + donec chan bool // closed when done + dt time.Duration + + src string + action string // "compile", "build", etc. + + tempDir string + err error +} + +// startTest +func startTest(dir, gofile string) *test { + t := &test{ + dir: dir, + gofile: gofile, + donec: make(chan bool, 1), + } + if toRun == nil { + toRun = make(chan *test, maxTests) + go runTests() + } + select { + case toRun <- t: + default: + panic("toRun buffer size (maxTests) is too small") + } + return t +} + +// runTests runs tests in parallel, but respecting the order they +// were enqueued on the toRun channel. +func runTests() { + for { + ratec <- true + t := <-toRun + go func() { + t.run() + <-ratec + }() + } +} + +var cwd, _ = os.Getwd() + +func (t *test) goFileName() string { + return filepath.Join(t.dir, t.gofile) +} + +func (t *test) goDirName() string { + return filepath.Join(t.dir, strings.Replace(t.gofile, ".go", ".dir", -1)) +} + +func goDirFiles(longdir string) (filter []os.FileInfo, err error) { + files, dirErr := ioutil.ReadDir(longdir) + if dirErr != nil { + return nil, dirErr + } + for _, gofile := range files { + if filepath.Ext(gofile.Name()) == ".go" { + filter = append(filter, gofile) + } + } + return +} + +var packageRE = regexp.MustCompile(`(?m)^package (\w+)`) + +func goDirPackages(longdir string) ([][]string, error) { + files, err := goDirFiles(longdir) + if err != nil { + return nil, err + } + var pkgs [][]string + m := make(map[string]int) + for _, file := range files { + name := file.Name() + data, err := ioutil.ReadFile(filepath.Join(longdir, name)) + if err != nil { + return nil, err + } + pkgname := packageRE.FindStringSubmatch(string(data)) + if pkgname == nil { + return nil, fmt.Errorf("cannot find package name in %s", name) + } + i, ok := m[pkgname[1]] + if !ok { + i = len(pkgs) + pkgs = append(pkgs, nil) + m[pkgname[1]] = i + } + pkgs[i] = append(pkgs[i], name) + } + return pkgs, nil +} + +type context struct { + GOOS string + GOARCH string +} + +// shouldTest looks for build tags in a source file and returns +// whether the file should be used according to the tags. +func shouldTest(src string, goos, goarch string) (ok bool, whyNot string) { + // Custom rule, treat js as equivalent to nacl. + if goarch == "js" { + goarch = "nacl" + } + + for _, line := range strings.Split(src, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "//") { + line = line[2:] + } else { + continue + } + line = strings.TrimSpace(line) + if len(line) == 0 || line[0] != '+' { + continue + } + ctxt := &context{ + GOOS: goos, + GOARCH: goarch, + } + words := strings.Fields(line) + if words[0] == "+build" { + ok := false + for _, word := range words[1:] { + if ctxt.match(word) { + ok = true + break + } + } + if !ok { + // no matching tag found. + return false, line + } + } + } + // no build tags + return true, "" +} + +func (ctxt *context) match(name string) bool { + if name == "" { + return false + } + if i := strings.Index(name, ","); i >= 0 { + // comma-separated list + return ctxt.match(name[:i]) && ctxt.match(name[i+1:]) + } + if strings.HasPrefix(name, "!!") { // bad syntax, reject always + return false + } + if strings.HasPrefix(name, "!") { // negation + return len(name) > 1 && !ctxt.match(name[1:]) + } + + // Tags must be letters, digits, underscores or dots. + // Unlike in Go identifiers, all digits are fine (e.g., "386"). + for _, c := range name { + if !unicode.IsLetter(c) && !unicode.IsDigit(c) && c != '_' && c != '.' { + return false + } + } + + if name == ctxt.GOOS || name == ctxt.GOARCH { + return true + } + + return false +} + +func init() { checkShouldTest() } + +// run runs a test. +func (t *test) run() { + start := time.Now() + defer func() { + t.dt = time.Since(start) + close(t.donec) + }() + + srcBytes, err := ioutil.ReadFile(t.goFileName()) + if err != nil { + t.err = err + return + } + t.src = string(srcBytes) + if t.src[0] == '\n' { + t.err = skipError("starts with newline") + return + } + + // Execution recipe stops at first blank line. + pos := strings.Index(t.src, "\n\n") + if pos == -1 { + t.err = errors.New("double newline not found") + return + } + action := t.src[:pos] + if nl := strings.Index(action, "\n"); nl >= 0 && strings.Contains(action[:nl], "+build") { + // skip first line + action = action[nl+1:] + } + if strings.HasPrefix(action, "//") { + action = action[2:] + } + + // Check for build constraints only up to the actual code. + pkgPos := strings.Index(t.src, "\npackage") + if pkgPos == -1 { + pkgPos = pos // some files are intentionally malformed + } + if ok, why := shouldTest(t.src[:pkgPos], goos, goarch); !ok { + t.action = "skip" + if *showSkips { + fmt.Printf("%-20s %-20s: %s\n", t.action, t.goFileName(), why) + } + return + } + + var args, flags []string + wantError := false + f := strings.Fields(action) + if len(f) > 0 { + action = f[0] + args = f[1:] + } + + // GOPHERJS: For now, only run with "run", "cmpout" actions, in "fixedbugs" dir. + switch action { + case "run", "cmpout": + if filepath.Clean(t.dir) != "fixedbugs" { + action = "skip" + } + case "rundircmpout", "compile", "compiledir", "build", "runoutput", + "rundir", "errorcheck", "errorcheckdir", "errorcheckoutput": + action = "skip" + } + + switch action { + case "rundircmpout": + action = "rundir" + t.action = "rundir" + case "cmpout": + action = "run" // the run case already looks for /.out files + fallthrough + case "compile", "compiledir", "build", "run", "runoutput", "rundir": + t.action = action + case "errorcheck", "errorcheckdir", "errorcheckoutput": + t.action = action + wantError = true + for len(args) > 0 && strings.HasPrefix(args[0], "-") { + if args[0] == "-0" { + wantError = false + } else { + flags = append(flags, args[0]) + } + args = args[1:] + } + case "skip": + t.action = "skip" + return + default: + t.err = skipError("skipped; unknown pattern: " + action) + t.action = "??" + return + } + + t.makeTempDir() + defer os.RemoveAll(t.tempDir) + + err = ioutil.WriteFile(filepath.Join(t.tempDir, t.gofile), srcBytes, 0644) + check(err) + + // A few tests (of things like the environment) require these to be set. + if os.Getenv("GOOS") == "" { + os.Setenv("GOOS", runtime.GOOS) + } + if os.Getenv("GOARCH") == "" { + os.Setenv("GOARCH", runtime.GOARCH) + } + + useTmp := true + runcmd := func(args ...string) ([]byte, error) { + cmd := exec.Command(args[0], args[1:]...) + var buf bytes.Buffer + cmd.Stdout = &buf + cmd.Stderr = &buf + if useTmp { + cmd.Dir = t.tempDir + cmd.Env = envForDir(cmd.Dir) + } + err := cmd.Run() + if err != nil { + err = fmt.Errorf("%s\n%s", err, buf.Bytes()) + } + return buf.Bytes(), err + } + + long := filepath.Join(cwd, t.goFileName()) + switch action { + default: + t.err = fmt.Errorf("unimplemented action %q", action) + + case "errorcheck": + cmdline := []string{"go", "tool", "compile", "-e", "-o", "a.o"} + cmdline = append(cmdline, flags...) + cmdline = append(cmdline, long) + out, err := runcmd(cmdline...) + if wantError { + if err == nil { + t.err = fmt.Errorf("compilation succeeded unexpectedly\n%s", out) + return + } + } else { + if err != nil { + t.err = err + return + } + } + if *updateErrors { + t.updateErrors(string(out), long) + } + t.err = t.errorCheck(string(out), long, t.gofile) + return + + case "compile": + _, t.err = compileFile(runcmd, long) + + case "compiledir": + // Compile all files in the directory in lexicographic order. + longdir := filepath.Join(cwd, t.goDirName()) + pkgs, err := goDirPackages(longdir) + if err != nil { + t.err = err + return + } + for _, gofiles := range pkgs { + _, t.err = compileInDir(runcmd, longdir, gofiles...) + if t.err != nil { + return + } + } + + case "errorcheckdir": + // errorcheck all files in lexicographic order + // useful for finding importing errors + longdir := filepath.Join(cwd, t.goDirName()) + pkgs, err := goDirPackages(longdir) + if err != nil { + t.err = err + return + } + for i, gofiles := range pkgs { + out, err := compileInDir(runcmd, longdir, gofiles...) + if i == len(pkgs)-1 { + if wantError && err == nil { + t.err = fmt.Errorf("compilation succeeded unexpectedly\n%s", out) + return + } else if !wantError && err != nil { + t.err = err + return + } + } else if err != nil { + t.err = err + return + } + var fullshort []string + for _, name := range gofiles { + fullshort = append(fullshort, filepath.Join(longdir, name), name) + } + t.err = t.errorCheck(string(out), fullshort...) + if t.err != nil { + break + } + } + + case "rundir": + // Compile all files in the directory in lexicographic order. + // then link as if the last file is the main package and run it + longdir := filepath.Join(cwd, t.goDirName()) + pkgs, err := goDirPackages(longdir) + if err != nil { + t.err = err + return + } + for i, gofiles := range pkgs { + _, err := compileInDir(runcmd, longdir, gofiles...) + if err != nil { + t.err = err + return + } + if i == len(pkgs)-1 { + err = linkFile(runcmd, gofiles[0]) + if err != nil { + t.err = err + return + } + var cmd []string + cmd = append(cmd, findExecCmd()...) + cmd = append(cmd, filepath.Join(t.tempDir, "a.exe")) + cmd = append(cmd, args...) + out, err := runcmd(cmd...) + if err != nil { + t.err = err + return + } + if strings.Replace(string(out), "\r\n", "\n", -1) != t.expectedOutput() { + t.err = fmt.Errorf("incorrect output\n%s", out) + } + } + } + + case "build": + _, err := runcmd("go", "build", "-o", "a.exe", long) + if err != nil { + t.err = err + } + + case "run": + useTmp = false + // GOPHERJS. + out, err := runcmd(append([]string{"gopherjs", "run", t.goFileName()}, args...)...) + if err != nil { + t.err = err + return + } + if strings.Replace(string(out), "\r\n", "\n", -1) != t.expectedOutput() { + t.err = fmt.Errorf("incorrect output\n%s", out) + } + + case "runoutput": + rungatec <- true + defer func() { + <-rungatec + }() + useTmp = false + out, err := runcmd(append([]string{"go", "run", t.goFileName()}, args...)...) + if err != nil { + t.err = err + return + } + tfile := filepath.Join(t.tempDir, "tmp__.go") + if err := ioutil.WriteFile(tfile, out, 0666); err != nil { + t.err = fmt.Errorf("write tempfile:%s", err) + return + } + out, err = runcmd("go", "run", tfile) + if err != nil { + t.err = err + return + } + if string(out) != t.expectedOutput() { + t.err = fmt.Errorf("incorrect output\n%s", out) + } + + case "errorcheckoutput": + useTmp = false + out, err := runcmd(append([]string{"go", "run", t.goFileName()}, args...)...) + if err != nil { + t.err = err + return + } + tfile := filepath.Join(t.tempDir, "tmp__.go") + err = ioutil.WriteFile(tfile, out, 0666) + if err != nil { + t.err = fmt.Errorf("write tempfile:%s", err) + return + } + cmdline := []string{"go", "tool", "compile", "-e", "-o", "a.o"} + cmdline = append(cmdline, flags...) + cmdline = append(cmdline, tfile) + out, err = runcmd(cmdline...) + if wantError { + if err == nil { + t.err = fmt.Errorf("compilation succeeded unexpectedly\n%s", out) + return + } + } else { + if err != nil { + t.err = err + return + } + } + t.err = t.errorCheck(string(out), tfile, "tmp__.go") + return + } +} + +var execCmd []string + +func findExecCmd() []string { + if execCmd != nil { + return execCmd + } + execCmd = []string{} // avoid work the second time + if goos == runtime.GOOS && goarch == runtime.GOARCH { + return execCmd + } + path, err := exec.LookPath(fmt.Sprintf("go_%s_%s_exec", goos, goarch)) + if err == nil { + execCmd = []string{path} + } + return execCmd +} + +func (t *test) String() string { + return filepath.Join(t.dir, t.gofile) +} + +func (t *test) makeTempDir() { + var err error + t.tempDir, err = ioutil.TempDir("", "") + check(err) +} + +func (t *test) expectedOutput() string { + filename := filepath.Join(t.dir, t.gofile) + filename = filename[:len(filename)-len(".go")] + filename += ".out" + b, _ := ioutil.ReadFile(filename) + return string(b) +} + +func splitOutput(out string) []string { + // gc error messages continue onto additional lines with leading tabs. + // Split the output at the beginning of each line that doesn't begin with a tab. + // lines are impossible to match so those are filtered out. + var res []string + for _, line := range strings.Split(out, "\n") { + if strings.HasSuffix(line, "\r") { // remove '\r', output by compiler on windows + line = line[:len(line)-1] + } + if strings.HasPrefix(line, "\t") { + res[len(res)-1] += "\n" + line + } else if strings.HasPrefix(line, "go tool") || strings.HasPrefix(line, "") { + continue + } else if strings.TrimSpace(line) != "" { + res = append(res, line) + } + } + return res +} + +func (t *test) errorCheck(outStr string, fullshort ...string) (err error) { + defer func() { + if *verbose && err != nil { + log.Printf("%s gc output:\n%s", t, outStr) + } + }() + var errs []error + out := splitOutput(outStr) + + // Cut directory name. + for i := range out { + for j := 0; j < len(fullshort); j += 2 { + full, short := fullshort[j], fullshort[j+1] + out[i] = strings.Replace(out[i], full, short, -1) + } + } + + var want []wantedError + for j := 0; j < len(fullshort); j += 2 { + full, short := fullshort[j], fullshort[j+1] + want = append(want, t.wantedErrors(full, short)...) + } + + for _, we := range want { + var errmsgs []string + errmsgs, out = partitionStrings(we.prefix, out) + if len(errmsgs) == 0 { + errs = append(errs, fmt.Errorf("%s:%d: missing error %q", we.file, we.lineNum, we.reStr)) + continue + } + matched := false + n := len(out) + for _, errmsg := range errmsgs { + if we.re.MatchString(errmsg) { + matched = true + } else { + out = append(out, errmsg) + } + } + if !matched { + errs = append(errs, fmt.Errorf("%s:%d: no match for %#q in:\n\t%s", we.file, we.lineNum, we.reStr, strings.Join(out[n:], "\n\t"))) + continue + } + } + + if len(out) > 0 { + errs = append(errs, fmt.Errorf("Unmatched Errors:")) + for _, errLine := range out { + errs = append(errs, fmt.Errorf("%s", errLine)) + } + } + + if len(errs) == 0 { + return nil + } + if len(errs) == 1 { + return errs[0] + } + var buf bytes.Buffer + fmt.Fprintf(&buf, "\n") + for _, err := range errs { + fmt.Fprintf(&buf, "%s\n", err.Error()) + } + return errors.New(buf.String()) +} + +func (t *test) updateErrors(out string, file string) { + // Read in source file. + src, err := ioutil.ReadFile(file) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return + } + lines := strings.Split(string(src), "\n") + // Remove old errors. + for i, ln := range lines { + pos := strings.Index(ln, " // ERROR ") + if pos >= 0 { + lines[i] = ln[:pos] + } + } + // Parse new errors. + errors := make(map[int]map[string]bool) + tmpRe := regexp.MustCompile(`autotmp_[0-9]+`) + for _, errStr := range splitOutput(out) { + colon1 := strings.Index(errStr, ":") + if colon1 < 0 || errStr[:colon1] != file { + continue + } + colon2 := strings.Index(errStr[colon1+1:], ":") + if colon2 < 0 { + continue + } + colon2 += colon1 + 1 + line, err := strconv.Atoi(errStr[colon1+1 : colon2]) + line-- + if err != nil || line < 0 || line >= len(lines) { + continue + } + msg := errStr[colon2+2:] + for _, r := range []string{`\`, `*`, `+`, `[`, `]`, `(`, `)`} { + msg = strings.Replace(msg, r, `\`+r, -1) + } + msg = strings.Replace(msg, `"`, `.`, -1) + msg = tmpRe.ReplaceAllLiteralString(msg, `autotmp_[0-9]+`) + if errors[line] == nil { + errors[line] = make(map[string]bool) + } + errors[line][msg] = true + } + // Add new errors. + for line, errs := range errors { + var sorted []string + for e := range errs { + sorted = append(sorted, e) + } + sort.Strings(sorted) + lines[line] += " // ERROR" + for _, e := range sorted { + lines[line] += fmt.Sprintf(` "%s$"`, e) + } + } + // Write new file. + err = ioutil.WriteFile(file, []byte(strings.Join(lines, "\n")), 0640) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return + } + // Polish. + exec.Command("go", "fmt", file).CombinedOutput() +} + +// matchPrefix reports whether s is of the form ^(.*/)?prefix(:|[), +// That is, it needs the file name prefix followed by a : or a [, +// and possibly preceded by a directory name. +func matchPrefix(s, prefix string) bool { + i := strings.Index(s, ":") + if i < 0 { + return false + } + j := strings.LastIndex(s[:i], "/") + s = s[j+1:] + if len(s) <= len(prefix) || s[:len(prefix)] != prefix { + return false + } + switch s[len(prefix)] { + case '[', ':': + return true + } + return false +} + +func partitionStrings(prefix string, strs []string) (matched, unmatched []string) { + for _, s := range strs { + if matchPrefix(s, prefix) { + matched = append(matched, s) + } else { + unmatched = append(unmatched, s) + } + } + return +} + +type wantedError struct { + reStr string + re *regexp.Regexp + lineNum int + file string + prefix string +} + +var ( + errRx = regexp.MustCompile(`// (?:GC_)?ERROR (.*)`) + errQuotesRx = regexp.MustCompile(`"([^"]*)"`) + lineRx = regexp.MustCompile(`LINE(([+-])([0-9]+))?`) +) + +func (t *test) wantedErrors(file, short string) (errs []wantedError) { + cache := make(map[string]*regexp.Regexp) + + src, _ := ioutil.ReadFile(file) + for i, line := range strings.Split(string(src), "\n") { + lineNum := i + 1 + if strings.Contains(line, "////") { + // double comment disables ERROR + continue + } + m := errRx.FindStringSubmatch(line) + if m == nil { + continue + } + all := m[1] + mm := errQuotesRx.FindAllStringSubmatch(all, -1) + if mm == nil { + log.Fatalf("%s:%d: invalid errchk line: %s", t.goFileName(), lineNum, line) + } + for _, m := range mm { + rx := lineRx.ReplaceAllStringFunc(m[1], func(m string) string { + n := lineNum + if strings.HasPrefix(m, "LINE+") { + delta, _ := strconv.Atoi(m[5:]) + n += delta + } else if strings.HasPrefix(m, "LINE-") { + delta, _ := strconv.Atoi(m[5:]) + n -= delta + } + return fmt.Sprintf("%s:%d", short, n) + }) + re := cache[rx] + if re == nil { + var err error + re, err = regexp.Compile(rx) + if err != nil { + log.Fatalf("%s:%d: invalid regexp \"%s\" in ERROR line: %v", t.goFileName(), lineNum, rx, err) + } + cache[rx] = re + } + prefix := fmt.Sprintf("%s:%d", short, lineNum) + errs = append(errs, wantedError{ + reStr: rx, + re: re, + prefix: prefix, + lineNum: lineNum, + file: short, + }) + } + } + + return +} + +// defaultRunOutputLimit returns the number of runoutput tests that +// can be executed in parallel. +func defaultRunOutputLimit() int { + const maxArmCPU = 2 + + cpu := runtime.NumCPU() + if runtime.GOARCH == "arm" && cpu > maxArmCPU { + cpu = maxArmCPU + } + return cpu +} + +// checkShouldTest runs sanity checks on the shouldTest function. +func checkShouldTest() { + assert := func(ok bool, _ string) { + if !ok { + panic("fail") + } + } + assertNot := func(ok bool, _ string) { assert(!ok, "") } + + // Simple tests. + assert(shouldTest("// +build linux", "linux", "arm")) + assert(shouldTest("// +build !windows", "linux", "arm")) + assertNot(shouldTest("// +build !windows", "windows", "amd64")) + + // A file with no build tags will always be tested. + assert(shouldTest("// This is a test.", "os", "arch")) + + // Build tags separated by a space are OR-ed together. + assertNot(shouldTest("// +build arm 386", "linux", "amd64")) + + // Build tags separated by a comma are AND-ed together. + assertNot(shouldTest("// +build !windows,!plan9", "windows", "amd64")) + assertNot(shouldTest("// +build !windows,!plan9", "plan9", "386")) + + // Build tags on multiple lines are AND-ed together. + assert(shouldTest("// +build !windows\n// +build amd64", "linux", "amd64")) + assertNot(shouldTest("// +build !windows\n// +build amd64", "windows", "amd64")) + + // Test that (!a OR !b) matches anything. + assert(shouldTest("// +build !windows !plan9", "windows", "amd64")) + + // GOPHERJS: Custom rule, test that don't run on nacl should also not run on js. + assertNot(shouldTest("// +build !nacl,!plan9,!windows", "darwin", "js")) +} + +// envForDir returns a copy of the environment +// suitable for running in the given directory. +// The environment is the current process's environment +// but with an updated $PWD, so that an os.Getwd in the +// child will be faster. +func envForDir(dir string) []string { + env := os.Environ() + for i, kv := range env { + if strings.HasPrefix(kv, "PWD=") { + env[i] = "PWD=" + dir + return env + } + } + env = append(env, "PWD="+dir) + return env +} + +func getenv(key, def string) string { + value := os.Getenv(key) + if value != "" { + return value + } + return def +}