Skip to content

[WIP] ci: parallelise the running of gopherjs tests #796

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
version: 2
jobs:
build:
working_directory: /go/src/github.com/gopherjs/gopherjs
docker:
- image: circleci/golang:1.10

environment:
SOURCE_MAP_SUPPORT: false

steps:
- run: |
git clone https://github.com/creationix/nvm.git $HOME/.nvm
cd $HOME/.nvm && git checkout v0.33.9
echo 'export NVM_DIR="$HOME/.nvm"' >> $BASH_ENV
echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"' >> $BASH_ENV

- run: |
nvm install 6.2.2 node
nvm alias default 6.2.2

- checkout

- run: |
node --version
go version
go env
nproc --all

- run: |
npm install --global node-gyp
cd node-syscall && node-gyp rebuild && mkdir -p $HOME/.node_libraries/ && cp build/Release/syscall.node $HOME/.node_libraries/syscall.node

- run: |
go get myitcv.io/cmd/concsh
go get -t ./...
go install github.com/gopherjs/gopherjs

- run:
no_output_timeout: 30m
command: |
diff -u <(echo -n) <(gofmt -d .)
go tool vet *.go # Go package in root directory.
for d in */; do echo $d; done | grep -v tests/ | grep -v third_party/ | xargs go tool vet # All subdirectories except "tests", "third_party".
diff -u <(echo -n) <(go list ./compiler/natives/src/...) # All those packages should have // +build js.
gopherjs install -v net/http # Should build successfully (can't run tests, since only client is supported).
ulimit -s 10000 && go run run_tests.go -debug -p 2
go test -v -race ./...
gopherjs test -v fmt # No minification should work.
134 changes: 0 additions & 134 deletions circle.yml

This file was deleted.

123 changes: 123 additions & 0 deletions run_tests.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
// +build ignore

package main

import (
"bufio"
"flag"
"fmt"
"go/build"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
)

const (
self = "github.com/gopherjs/gopherjs"
)

var (
fDebug = flag.Bool("debug", false, "include debug output")
fParallelism = flag.Int("p", runtime.NumCPU(), "number of pkgs to test in parallel")
)

func main() {
flag.Parse()

if err := run(); err != nil {
fmt.Fprintf(os.Stderr, "Failed to run tests: %v\n", err)
os.Exit(1)
}
}

func run() error {

bpkg, err := build.Import(self, ".", build.FindOnly)
if err != nil {
return fmt.Errorf("failed to get directory for import %v: %v", self, err)
}

exPath := filepath.Join(bpkg.Dir, "std_test_pkg_exclusions")

exFi, err := os.Open(exPath)
if err != nil {
return fmt.Errorf("error opening %v: %v", exPath, err)
}

excls := make(map[string]bool)

{
sc := bufio.NewScanner(exFi)
for sc.Scan() {
line := strings.TrimSpace(sc.Text())

if strings.HasPrefix(line, "#") {
continue
}

excls[line] = true
}
if err := sc.Err(); err != nil {
return fmt.Errorf("failed to line scan %v: %v", exPath, err)
}
}

cmd := exec.Command("go", "list", "std")
stdListOut, err := cmd.Output()
if err != nil {
return fmt.Errorf("failed to %v: %v", strings.Join(cmd.Args, " "), err)
}

tests := []string{
"github.com/gopherjs/gopherjs/tests",
"github.com/gopherjs/gopherjs/tests/main",
"github.com/gopherjs/gopherjs/js",
}

{
sc := bufio.NewScanner(strings.NewReader(string(stdListOut)))
for sc.Scan() {
line := strings.TrimSpace(sc.Text())

if strings.HasPrefix(line, "#") {
continue
}

if !excls[line] {
tests = append(tests, line)
}
}
if err := sc.Err(); err != nil {
return fmt.Errorf("failed to line scan %q: %v", strings.Join(cmd.Args, " "), err)
}
}

var cmds []string

for _, t := range tests {
cmds = append(cmds, fmt.Sprintf("gopherjs test -m %v\n", t))
}

p := *fParallelism

debugf("running tests with parallelism %v\n", p)

testCmd := exec.Command("concsh", "-conc", fmt.Sprintf("%v", p))
testCmd.Stdin = strings.NewReader(strings.Join(cmds, ""))
testCmd.Stdout = os.Stdout
testCmd.Stderr = os.Stderr

if err := testCmd.Run(); err != nil {
return fmt.Errorf("test process exited with an error: %v", err)
}

return nil
}

func debugf(format string, args ...interface{}) {
if *fDebug {
fmt.Fprintf(os.Stderr, format, args...)
}
}
80 changes: 80 additions & 0 deletions std_test_pkg_exclusions
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
# these are go list std packages we want to exclude from our tests
context
crypto
crypto/internal/cipherhw
crypto/tls
crypto/x509/pkix
debug/gosym
debug/plan9obj
encoding
go/build
go/importer
go/internal/gccgoimporter
go/internal/gcimporter
go/internal/srcimporter
go/types
hash
image/color/palette
image/internal/imageutil
internal/cpu
internal/nettrace
internal/poll
internal/race
internal/singleflight
internal/syscall/unix
internal/syscall/windows
internal/syscall/windows/registry
internal/syscall/windows/sysdll
internal/testenv
internal/testlog
internal/trace
log
log/syslog
net
net/http
net/http/cgi
net/http/httptest
net/http/httptrace
net/http/httputil
net/http/internal
net/http/pprof
net/internal/socktest
net/rpc
net/smtp
os
os/exec
os/signal
os/signal/internal/pty
plugin
runtime
runtime/cgo
runtime/debug
runtime/internal/atomic
runtime/internal/sys
runtime/pprof
runtime/pprof/internal/profile
runtime/race
runtime/trace
syscall
testing
testing/internal/testdeps
testing/iotest
unsafe
vendor/golang_org/x/crypto/chacha20poly1305
vendor/golang_org/x/crypto/chacha20poly1305/internal/chacha20
vendor/golang_org/x/crypto/cryptobyte
vendor/golang_org/x/crypto/cryptobyte/asn1
vendor/golang_org/x/crypto/curve25519
vendor/golang_org/x/crypto/poly1305
vendor/golang_org/x/net/http2/hpack
vendor/golang_org/x/net/idna
vendor/golang_org/x/net/internal/nettest
vendor/golang_org/x/net/lex/httplex
vendor/golang_org/x/net/nettest
vendor/golang_org/x/net/proxy
vendor/golang_org/x/text/secure
vendor/golang_org/x/text/secure/bidirule
vendor/golang_org/x/text/transform
vendor/golang_org/x/text/unicode
vendor/golang_org/x/text/unicode/bidi
vendor/golang_org/x/text/unicode/norm