ci: parallelise the running of gopherjs tests

myitcv · myitcv · commit 060631220113 · 2018-04-18T16:02:07.000+01:00
Currently we run the gopherjs tests in sequence; this is limited to a
single CPU core because we ultimately run the tests via Node. We can
improve on this by concurrently running gopherjs test per package.

Also implemented a fix discussed in a previous issue/PR to make our list
of package exclusions explicit, via std_test_pkg_exclusions
diff --git a/circle.yml b/circle.yml
@@ -7,6 +7,9 @@ machine:
 dependencies:
   pre:
     - cd /usr/local && sudo rm -rf go && curl https://storage.googleapis.com/golang/go1.10.linux-amd64.tar.gz | sudo tar -xz && sudo chmod a+w go/src/path/filepath
+    - go get myitcv.io/cmd/concsh
+    - which concsh
+    - nproc --all
   post:
     - mv ./gopherjs $HOME/bin
     - npm install --global node-gyp
@@ -19,116 +22,6 @@ test:
     - for d in */; do echo $d; done | grep -v tests/ | grep -v third_party/ | xargs go tool vet # All subdirectories except "tests", "third_party".
     - diff -u <(echo -n) <(go list ./compiler/natives/src/...) # All those packages should have // +build js.
     - gopherjs install -v net/http # Should build successfully (can't run tests, since only client is supported).
-    - >
-      ulimit -s 10000 && gopherjs test --minify -v --short
-      github.com/gopherjs/gopherjs/tests
-      github.com/gopherjs/gopherjs/tests/main
-      github.com/gopherjs/gopherjs/js
-      archive/tar
-      archive/zip
-      bufio
-      bytes
-      compress/bzip2
-      compress/flate
-      compress/gzip
-      compress/lzw
-      compress/zlib
-      container/heap
-      container/list
-      container/ring
-      crypto/aes
-      crypto/cipher
-      crypto/des
-      crypto/dsa
-      crypto/ecdsa
-      crypto/elliptic
-      crypto/hmac
-      crypto/md5
-      crypto/rand
-      crypto/rc4
-      crypto/rsa
-      crypto/sha1
-      crypto/sha256
-      crypto/sha512
-      crypto/subtle
-      crypto/x509
-      database/sql
-      database/sql/driver
-      debug/dwarf
-      debug/elf
-      debug/macho
-      debug/pe
-      encoding/ascii85
-      encoding/asn1
-      encoding/base32
-      encoding/base64
-      encoding/binary
-      encoding/csv
-      encoding/gob
-      encoding/hex
-      encoding/json
-      encoding/pem
-      encoding/xml
-      errors
-      expvar
-      flag
-      fmt
-      go/ast
-      go/constant
-      go/doc
-      go/format
-      go/parser
-      go/printer
-      go/scanner
-      go/token
-      hash/adler32
-      hash/crc32
-      hash/crc64
-      hash/fnv
-      html
-      html/template
-      image
-      image/color
-      image/draw
-      image/gif
-      image/jpeg
-      image/png
-      index/suffixarray
-      io
-      io/ioutil
-      math
-      math/big
-      math/bits
-      math/cmplx
-      math/rand
-      mime
-      mime/multipart
-      mime/quotedprintable
-      net/http/cookiejar
-      net/http/fcgi
-      net/mail
-      net/rpc/jsonrpc
-      net/textproto
-      net/url
-      os/user
-      path
-      path/filepath
-      reflect
-      regexp
-      regexp/syntax
-      sort
-      strconv
-      strings
-      sync
-      sync/atomic
-      testing/quick
-      text/scanner
-      text/tabwriter
-      text/template
-      text/template/parse
-      time
-      unicode
-      unicode/utf16
-      unicode/utf8
+    - go run run_tests.go -debug
     - go test -v -race ./...
     - gopherjs test -v fmt # No minification should work.
diff --git a/run_tests.go b/run_tests.go
@@ -0,0 +1,128 @@
+// +build ignore
+
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"go/build"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+const (
+	ciParallelism = 2
+	self          = "github.com/gopherjs/gopherjs"
+)
+
+var (
+	fDebug = flag.Bool("debug", false, "include debug output")
+)
+
+func main() {
+	flag.Parse()
+
+	if err := run(); err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to run tests: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func run() error {
+
+	bpkg, err := build.Import(self, ".", build.FindOnly)
+	if err != nil {
+		return fmt.Errorf("failed to get directory for import %v: %v", self, err)
+	}
+
+	exPath := filepath.Join(bpkg.Dir, "std_test_pkg_exclusions")
+
+	exFi, err := os.Open(exPath)
+	if err != nil {
+		return fmt.Errorf("error opening %v: %v", exPath, err)
+	}
+
+	excls := make(map[string]bool)
+
+	{
+		sc := bufio.NewScanner(exFi)
+		for sc.Scan() {
+			line := strings.TrimSpace(sc.Text())
+
+			if strings.HasPrefix(line, "#") {
+				continue
+			}
+
+			excls[line] = true
+		}
+		if err := sc.Err(); err != nil {
+			return fmt.Errorf("failed to line scan %v: %v", exPath, err)
+		}
+	}
+
+	cmd := exec.Command("go", "list", "std")
+	stdListOut, err := cmd.Output()
+	if err != nil {
+		return fmt.Errorf("failed to %v: %v", strings.Join(cmd.Args, " "), err)
+	}
+
+	tests := []string{
+		"github.com/gopherjs/gopherjs/tests",
+		"github.com/gopherjs/gopherjs/tests/main",
+		"github.com/gopherjs/gopherjs/js",
+	}
+
+	{
+		sc := bufio.NewScanner(strings.NewReader(string(stdListOut)))
+		for sc.Scan() {
+			line := strings.TrimSpace(sc.Text())
+
+			if strings.HasPrefix(line, "#") {
+				continue
+			}
+
+			if !excls[line] {
+				tests = append(tests, line)
+			}
+		}
+		if err := sc.Err(); err != nil {
+			return fmt.Errorf("failed to line scan %q: %v", strings.Join(cmd.Args, " "), err)
+		}
+	}
+
+	var cmds []string
+
+	for _, t := range tests {
+		cmds = append(cmds, fmt.Sprintf("gopherjs test -m %v\n", t))
+	}
+
+	p := runtime.NumCPU()
+
+	if v, ok := os.LookupEnv("CI"); ok && v == "TRUE" {
+		// because Circle doesn't seem very happy when we flood it with work
+		p = ciParallelism
+	}
+
+	debugf("running tests with parallelism %v\n", p)
+
+	testCmd := exec.Command("concsh", "-conc", fmt.Sprintf("%v", p))
+	testCmd.Stdin = strings.NewReader(strings.Join(cmds, ""))
+	testCmd.Stdout = os.Stdout
+	testCmd.Stderr = os.Stderr
+
+	if err := testCmd.Run(); err != nil {
+		return fmt.Errorf("test process exited with an error: %v", err)
+	}
+
+	return nil
+}
+
+func debugf(format string, args ...interface{}) {
+	if *fDebug {
+		fmt.Fprintf(os.Stderr, format, args...)
+	}
+}
diff --git a/std_test_pkg_exclusions b/std_test_pkg_exclusions
@@ -0,0 +1,80 @@
+# these are go list std packages we want to exclude from our tests
+context
+crypto
+crypto/internal/cipherhw
+crypto/tls
+crypto/x509/pkix
+debug/gosym
+debug/plan9obj
+encoding
+go/build
+go/importer
+go/internal/gccgoimporter
+go/internal/gcimporter
+go/internal/srcimporter
+go/types
+hash
+image/color/palette
+image/internal/imageutil
+internal/cpu
+internal/nettrace
+internal/poll
+internal/race
+internal/singleflight
+internal/syscall/unix
+internal/syscall/windows
+internal/syscall/windows/registry
+internal/syscall/windows/sysdll
+internal/testenv
+internal/testlog
+internal/trace
+log
+log/syslog
+net
+net/http
+net/http/cgi
+net/http/httptest
+net/http/httptrace
+net/http/httputil
+net/http/internal
+net/http/pprof
+net/internal/socktest
+net/rpc
+net/smtp
+os
+os/exec
+os/signal
+os/signal/internal/pty
+plugin
+runtime
+runtime/cgo
+runtime/debug
+runtime/internal/atomic
+runtime/internal/sys
+runtime/pprof
+runtime/pprof/internal/profile
+runtime/race
+runtime/trace
+syscall
+testing
+testing/internal/testdeps
+testing/iotest
+unsafe
+vendor/golang_org/x/crypto/chacha20poly1305
+vendor/golang_org/x/crypto/chacha20poly1305/internal/chacha20
+vendor/golang_org/x/crypto/cryptobyte
+vendor/golang_org/x/crypto/cryptobyte/asn1
+vendor/golang_org/x/crypto/curve25519
+vendor/golang_org/x/crypto/poly1305
+vendor/golang_org/x/net/http2/hpack
+vendor/golang_org/x/net/idna
+vendor/golang_org/x/net/internal/nettest
+vendor/golang_org/x/net/lex/httplex
+vendor/golang_org/x/net/nettest
+vendor/golang_org/x/net/proxy
+vendor/golang_org/x/text/secure
+vendor/golang_org/x/text/secure/bidirule
+vendor/golang_org/x/text/transform
+vendor/golang_org/x/text/unicode
+vendor/golang_org/x/text/unicode/bidi
+vendor/golang_org/x/text/unicode/norm
