Skip to content

Commit 20a0780

Browse files
myitcvmyitcv
committed
ci: parallelise the running of gopherjs tests
Currently we run the gopherjs tests in sequence; this is limited to a single CPU core because we ultimately run the tests via Node. We can improve on this by concurrently running gopherjs test per package. Also implemented a fix discussed in a previous issue/PR to make our list of package exclusions explicit, via std_test_pkg_exclusions
1 parent 1e1ddc5 commit 20a0780

File tree

4 files changed

+234
-110
lines changed

4 files changed

+234
-110
lines changed

‎.circleci/config.yml

Lines changed: 2 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ jobs:
3232
cd node-syscall && node-gyp rebuild && mkdir -p $HOME/.node_libraries/ && cp build/Release/syscall.node $HOME/.node_libraries/syscall.node
3333
3434
- run: |
35+
go get myitcv.io/cmd/concsh
3536
go get -t ./...
3637
go install github.com/gopherjs/gopherjs
3738
@@ -43,115 +44,6 @@ jobs:
4344
for d in */; do echo $d; done | grep -v tests/ | grep -v third_party/ | xargs go tool vet # All subdirectories except "tests", "third_party".
4445
diff -u <(echo -n) <(go list ./compiler/natives/src/...) # All those packages should have // +build js.
4546
gopherjs install -v net/http # Should build successfully (can't run tests, since only client is supported).
46-
ulimit -s 10000 && gopherjs test --minify -v --short \
47-
github.com/gopherjs/gopherjs/tests \
48-
github.com/gopherjs/gopherjs/tests/main \
49-
github.com/gopherjs/gopherjs/js \
50-
archive/tar \
51-
archive/zip \
52-
bufio \
53-
bytes \
54-
compress/bzip2 \
55-
compress/flate \
56-
compress/gzip \
57-
compress/lzw \
58-
compress/zlib \
59-
container/heap \
60-
container/list \
61-
container/ring \
62-
crypto/aes \
63-
crypto/cipher \
64-
crypto/des \
65-
crypto/dsa \
66-
crypto/ecdsa \
67-
crypto/elliptic \
68-
crypto/hmac \
69-
crypto/md5 \
70-
crypto/rand \
71-
crypto/rc4 \
72-
crypto/rsa \
73-
crypto/sha1 \
74-
crypto/sha256 \
75-
crypto/sha512 \
76-
crypto/subtle \
77-
crypto/x509 \
78-
database/sql \
79-
database/sql/driver \
80-
debug/dwarf \
81-
debug/elf \
82-
debug/macho \
83-
debug/pe \
84-
encoding/ascii85 \
85-
encoding/asn1 \
86-
encoding/base32 \
87-
encoding/base64 \
88-
encoding/binary \
89-
encoding/csv \
90-
encoding/gob \
91-
encoding/hex \
92-
encoding/json \
93-
encoding/pem \
94-
encoding/xml \
95-
errors \
96-
expvar \
97-
flag \
98-
fmt \
99-
go/ast \
100-
go/constant \
101-
go/doc \
102-
go/format \
103-
go/parser \
104-
go/printer \
105-
go/scanner \
106-
go/token \
107-
hash/adler32 \
108-
hash/crc32 \
109-
hash/crc64 \
110-
hash/fnv \
111-
html \
112-
html/template \
113-
image \
114-
image/color \
115-
image/draw \
116-
image/gif \
117-
image/jpeg \
118-
image/png \
119-
index/suffixarray \
120-
io \
121-
io/ioutil \
122-
math \
123-
math/big \
124-
math/bits \
125-
math/cmplx \
126-
math/rand \
127-
mime \
128-
mime/multipart \
129-
mime/quotedprintable \
130-
net/http/cookiejar \
131-
net/http/fcgi \
132-
net/mail \
133-
net/rpc/jsonrpc \
134-
net/textproto \
135-
net/url \
136-
os/user \
137-
path \
138-
path/filepath \
139-
reflect \
140-
regexp \
141-
regexp/syntax \
142-
sort \
143-
strconv \
144-
strings \
145-
sync \
146-
sync/atomic \
147-
testing/quick \
148-
text/scanner \
149-
text/tabwriter \
150-
text/template \
151-
text/template/parse \
152-
time \
153-
unicode \
154-
unicode/utf16 \
155-
unicode/utf8
47+
ulimit -s 10000 && go run run_tests.go -debug -p 2
15648
go test -v -race ./...
15749
gopherjs test -v fmt # No minification should work.

‎.travis.yml

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
language: go
2+
3+
go:
4+
- 1.10.x
5+
6+
env:
7+
- SOURCE_MAP_SUPPORT=false
8+
9+
go_import_path: github.com/gopherjs/gopherjs
10+
11+
before_script:
12+
- go get myitcv.io/cmd/concsh
13+
- which concsh
14+
- nproc --all
15+
- npm install --global node-gyp
16+
- cd node-syscall && node-gyp rebuild && mkdir -p ~/.node_libraries/ && cp build/Release/syscall.node ~/.node_libraries/syscall.node
17+
18+
install: true # no-op
19+
20+
script:
21+
- diff -u <(echo -n) <(gofmt -d .)
22+
- go tool vet *.go # Go package in root directory.
23+
- for d in */; do echo $d; done | grep -v tests/ | grep -v third_party/ | xargs go tool vet # All subdirectories except "tests", "third_party".
24+
- diff -u <(echo -n) <(go list ./compiler/natives/src/...) # All those packages should have // +build js.
25+
- gopherjs install -v net/http # Should build successfully (can't run tests, since only client is supported).
26+
- go run run_tests.go -debug -p 2
27+
- go test -v -race ./...
28+
- gopherjs test -v fmt # No minification should work.

‎run_tests.go

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
// +build ignore
2+
3+
package main
4+
5+
import (
6+
"bufio"
7+
"flag"
8+
"fmt"
9+
"go/build"
10+
"os"
11+
"os/exec"
12+
"path/filepath"
13+
"runtime"
14+
"strings"
15+
)
16+
17+
const (
18+
ciParallelism = 2
19+
self = "github.com/gopherjs/gopherjs"
20+
)
21+
22+
var (
23+
fDebug = flag.Bool("debug", false, "include debug output")
24+
fParallelism = flag.Int("p", runtime.NumCPU(), "number of pkgs to test in parallel")
25+
)
26+
27+
func main() {
28+
flag.Parse()
29+
30+
if err := run(); err != nil {
31+
fmt.Fprintf(os.Stderr, "Failed to run tests: %v\n", err)
32+
os.Exit(1)
33+
}
34+
}
35+
36+
func run() error {
37+
38+
bpkg, err := build.Import(self, ".", build.FindOnly)
39+
if err != nil {
40+
return fmt.Errorf("failed to get directory for import %v: %v", self, err)
41+
}
42+
43+
exPath := filepath.Join(bpkg.Dir, "std_test_pkg_exclusions")
44+
45+
exFi, err := os.Open(exPath)
46+
if err != nil {
47+
return fmt.Errorf("error opening %v: %v", exPath, err)
48+
}
49+
50+
excls := make(map[string]bool)
51+
52+
{
53+
sc := bufio.NewScanner(exFi)
54+
for sc.Scan() {
55+
line := strings.TrimSpace(sc.Text())
56+
57+
if strings.HasPrefix(line, "#") {
58+
continue
59+
}
60+
61+
excls[line] = true
62+
}
63+
if err := sc.Err(); err != nil {
64+
return fmt.Errorf("failed to line scan %v: %v", exPath, err)
65+
}
66+
}
67+
68+
cmd := exec.Command("go", "list", "std")
69+
stdListOut, err := cmd.Output()
70+
if err != nil {
71+
return fmt.Errorf("failed to %v: %v", strings.Join(cmd.Args, " "), err)
72+
}
73+
74+
tests := []string{
75+
"github.com/gopherjs/gopherjs/tests",
76+
"github.com/gopherjs/gopherjs/tests/main",
77+
"github.com/gopherjs/gopherjs/js",
78+
}
79+
80+
{
81+
sc := bufio.NewScanner(strings.NewReader(string(stdListOut)))
82+
for sc.Scan() {
83+
line := strings.TrimSpace(sc.Text())
84+
85+
if strings.HasPrefix(line, "#") {
86+
continue
87+
}
88+
89+
if !excls[line] {
90+
tests = append(tests, line)
91+
}
92+
}
93+
if err := sc.Err(); err != nil {
94+
return fmt.Errorf("failed to line scan %q: %v", strings.Join(cmd.Args, " "), err)
95+
}
96+
}
97+
98+
var cmds []string
99+
100+
for _, t := range tests {
101+
cmds = append(cmds, fmt.Sprintf("gopherjs test -m %v\n", t))
102+
}
103+
104+
p := *fParallelism
105+
106+
debugf("running tests with parallelism %v\n", p)
107+
108+
testCmd := exec.Command("concsh", "-conc", fmt.Sprintf("%v", p))
109+
testCmd.Stdin = strings.NewReader(strings.Join(cmds, ""))
110+
testCmd.Stdout = os.Stdout
111+
testCmd.Stderr = os.Stderr
112+
113+
if err := testCmd.Run(); err != nil {
114+
return fmt.Errorf("test process exited with an error: %v", err)
115+
}
116+
117+
return nil
118+
}
119+
120+
func debugf(format string, args ...interface{}) {
121+
if *fDebug {
122+
fmt.Fprintf(os.Stderr, format, args...)
123+
}
124+
}

‎std_test_pkg_exclusions

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# these are go list std packages we want to exclude from our tests
2+
context
3+
crypto
4+
crypto/internal/cipherhw
5+
crypto/tls
6+
crypto/x509/pkix
7+
debug/gosym
8+
debug/plan9obj
9+
encoding
10+
go/build
11+
go/importer
12+
go/internal/gccgoimporter
13+
go/internal/gcimporter
14+
go/internal/srcimporter
15+
go/types
16+
hash
17+
image/color/palette
18+
image/internal/imageutil
19+
internal/cpu
20+
internal/nettrace
21+
internal/poll
22+
internal/race
23+
internal/singleflight
24+
internal/syscall/unix
25+
internal/syscall/windows
26+
internal/syscall/windows/registry
27+
internal/syscall/windows/sysdll
28+
internal/testenv
29+
internal/testlog
30+
internal/trace
31+
log
32+
log/syslog
33+
net
34+
net/http
35+
net/http/cgi
36+
net/http/httptest
37+
net/http/httptrace
38+
net/http/httputil
39+
net/http/internal
40+
net/http/pprof
41+
net/internal/socktest
42+
net/rpc
43+
net/smtp
44+
os
45+
os/exec
46+
os/signal
47+
os/signal/internal/pty
48+
plugin
49+
runtime
50+
runtime/cgo
51+
runtime/debug
52+
runtime/internal/atomic
53+
runtime/internal/sys
54+
runtime/pprof
55+
runtime/pprof/internal/profile
56+
runtime/race
57+
runtime/trace
58+
syscall
59+
testing
60+
testing/internal/testdeps
61+
testing/iotest
62+
unsafe
63+
vendor/golang_org/x/crypto/chacha20poly1305
64+
vendor/golang_org/x/crypto/chacha20poly1305/internal/chacha20
65+
vendor/golang_org/x/crypto/cryptobyte
66+
vendor/golang_org/x/crypto/cryptobyte/asn1
67+
vendor/golang_org/x/crypto/curve25519
68+
vendor/golang_org/x/crypto/poly1305
69+
vendor/golang_org/x/net/http2/hpack
70+
vendor/golang_org/x/net/idna
71+
vendor/golang_org/x/net/internal/nettest
72+
vendor/golang_org/x/net/lex/httplex
73+
vendor/golang_org/x/net/nettest
74+
vendor/golang_org/x/net/proxy
75+
vendor/golang_org/x/text/secure
76+
vendor/golang_org/x/text/secure/bidirule
77+
vendor/golang_org/x/text/transform
78+
vendor/golang_org/x/text/unicode
79+
vendor/golang_org/x/text/unicode/bidi
80+
vendor/golang_org/x/text/unicode/norm

0 commit comments

Comments
 (0)