From 1e1ddc5726d2d00b1443198579d590936df7889f Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Wed, 18 Apr 2018 18:20:35 +0100 Subject: [PATCH 1/2] ci: upgrade to CircleCI 2 --- .circleci/config.yml | 157 +++++++++++++++++++++++++++++++++++++++++++ circle.yml | 134 ------------------------------------ 2 files changed, 157 insertions(+), 134 deletions(-) create mode 100644 .circleci/config.yml delete mode 100644 circle.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..1fbc65a56 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,157 @@ +version: 2 +jobs: + build: + working_directory: /go/src/github.com/gopherjs/gopherjs + docker: + - image: circleci/golang:1.10 + + environment: + SOURCE_MAP_SUPPORT: false + + steps: + - run: | + git clone https://github.com/creationix/nvm.git $HOME/.nvm + cd $HOME/.nvm && git checkout v0.33.9 + echo 'export NVM_DIR="$HOME/.nvm"' >> $BASH_ENV + echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"' >> $BASH_ENV + + - run: | + nvm install 6.2.2 node + nvm alias default 6.2.2 + + - checkout + + - run: | + node --version + go version + go env + nproc --all + + - run: | + npm install --global node-gyp + cd node-syscall && node-gyp rebuild && mkdir -p $HOME/.node_libraries/ && cp build/Release/syscall.node $HOME/.node_libraries/syscall.node + + - run: | + go get -t ./... + go install github.com/gopherjs/gopherjs + + - run: + no_output_timeout: 30m + command: | + diff -u <(echo -n) <(gofmt -d .) + go tool vet *.go # Go package in root directory. + for d in */; do echo $d; done | grep -v tests/ | grep -v third_party/ | xargs go tool vet # All subdirectories except "tests", "third_party". + diff -u <(echo -n) <(go list ./compiler/natives/src/...) # All those packages should have // +build js. + gopherjs install -v net/http # Should build successfully (can't run tests, since only client is supported). + ulimit -s 10000 && gopherjs test --minify -v --short \ + github.com/gopherjs/gopherjs/tests \ + github.com/gopherjs/gopherjs/tests/main \ + github.com/gopherjs/gopherjs/js \ + archive/tar \ + archive/zip \ + bufio \ + bytes \ + compress/bzip2 \ + compress/flate \ + compress/gzip \ + compress/lzw \ + compress/zlib \ + container/heap \ + container/list \ + container/ring \ + crypto/aes \ + crypto/cipher \ + crypto/des \ + crypto/dsa \ + crypto/ecdsa \ + crypto/elliptic \ + crypto/hmac \ + crypto/md5 \ + crypto/rand \ + crypto/rc4 \ + crypto/rsa \ + crypto/sha1 \ + crypto/sha256 \ + crypto/sha512 \ + crypto/subtle \ + crypto/x509 \ + database/sql \ + database/sql/driver \ + debug/dwarf \ + debug/elf \ + debug/macho \ + debug/pe \ + encoding/ascii85 \ + encoding/asn1 \ + encoding/base32 \ + encoding/base64 \ + encoding/binary \ + encoding/csv \ + encoding/gob \ + encoding/hex \ + encoding/json \ + encoding/pem \ + encoding/xml \ + errors \ + expvar \ + flag \ + fmt \ + go/ast \ + go/constant \ + go/doc \ + go/format \ + go/parser \ + go/printer \ + go/scanner \ + go/token \ + hash/adler32 \ + hash/crc32 \ + hash/crc64 \ + hash/fnv \ + html \ + html/template \ + image \ + image/color \ + image/draw \ + image/gif \ + image/jpeg \ + image/png \ + index/suffixarray \ + io \ + io/ioutil \ + math \ + math/big \ + math/bits \ + math/cmplx \ + math/rand \ + mime \ + mime/multipart \ + mime/quotedprintable \ + net/http/cookiejar \ + net/http/fcgi \ + net/mail \ + net/rpc/jsonrpc \ + net/textproto \ + net/url \ + os/user \ + path \ + path/filepath \ + reflect \ + regexp \ + regexp/syntax \ + sort \ + strconv \ + strings \ + sync \ + sync/atomic \ + testing/quick \ + text/scanner \ + text/tabwriter \ + text/template \ + text/template/parse \ + time \ + unicode \ + unicode/utf16 \ + unicode/utf8 + go test -v -race ./... + gopherjs test -v fmt # No minification should work. diff --git a/circle.yml b/circle.yml deleted file mode 100644 index f6ee89045..000000000 --- a/circle.yml +++ /dev/null @@ -1,134 +0,0 @@ -machine: - node: - version: 6.2.2 - environment: - SOURCE_MAP_SUPPORT: false - -dependencies: - pre: - - cd /usr/local && sudo rm -rf go && curl https://storage.googleapis.com/golang/go1.10.linux-amd64.tar.gz | sudo tar -xz && sudo chmod a+w go/src/path/filepath - post: - - mv ./gopherjs $HOME/bin - - npm install --global node-gyp - - cd node-syscall && node-gyp rebuild && mkdir -p ~/.node_libraries/ && cp build/Release/syscall.node ~/.node_libraries/syscall.node - -test: - override: - - diff -u <(echo -n) <(gofmt -d .) - - go tool vet *.go # Go package in root directory. - - for d in */; do echo $d; done | grep -v tests/ | grep -v third_party/ | xargs go tool vet # All subdirectories except "tests", "third_party". - - diff -u <(echo -n) <(go list ./compiler/natives/src/...) # All those packages should have // +build js. - - gopherjs install -v net/http # Should build successfully (can't run tests, since only client is supported). - - > - ulimit -s 10000 && gopherjs test --minify -v --short - github.com/gopherjs/gopherjs/tests - github.com/gopherjs/gopherjs/tests/main - github.com/gopherjs/gopherjs/js - archive/tar - archive/zip - bufio - bytes - compress/bzip2 - compress/flate - compress/gzip - compress/lzw - compress/zlib - container/heap - container/list - container/ring - crypto/aes - crypto/cipher - crypto/des - crypto/dsa - crypto/ecdsa - crypto/elliptic - crypto/hmac - crypto/md5 - crypto/rand - crypto/rc4 - crypto/rsa - crypto/sha1 - crypto/sha256 - crypto/sha512 - crypto/subtle - crypto/x509 - database/sql - database/sql/driver - debug/dwarf - debug/elf - debug/macho - debug/pe - encoding/ascii85 - encoding/asn1 - encoding/base32 - encoding/base64 - encoding/binary - encoding/csv - encoding/gob - encoding/hex - encoding/json - encoding/pem - encoding/xml - errors - expvar - flag - fmt - go/ast - go/constant - go/doc - go/format - go/parser - go/printer - go/scanner - go/token - hash/adler32 - hash/crc32 - hash/crc64 - hash/fnv - html - html/template - image - image/color - image/draw - image/gif - image/jpeg - image/png - index/suffixarray - io - io/ioutil - math - math/big - math/bits - math/cmplx - math/rand - mime - mime/multipart - mime/quotedprintable - net/http/cookiejar - net/http/fcgi - net/mail - net/rpc/jsonrpc - net/textproto - net/url - os/user - path - path/filepath - reflect - regexp - regexp/syntax - sort - strconv - strings - sync - sync/atomic - testing/quick - text/scanner - text/tabwriter - text/template - text/template/parse - time - unicode - unicode/utf16 - unicode/utf8 - - go test -v -race ./... - - gopherjs test -v fmt # No minification should work. From 4dd3bfa80c409bc90817ae1e2b446413e1e92488 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Wed, 18 Apr 2018 10:37:28 +0100 Subject: [PATCH 2/2] ci: parallelise the running of gopherjs tests Currently we run the gopherjs tests in sequence; this is limited to a single CPU core because we ultimately run the tests via Node. We can improve on this by concurrently running gopherjs test per package. Also implemented a fix discussed in a previous issue/PR to make our list of package exclusions explicit, via std_test_pkg_exclusions --- .circleci/config.yml | 112 +----------------------------------- run_tests.go | 123 ++++++++++++++++++++++++++++++++++++++++ std_test_pkg_exclusions | 80 ++++++++++++++++++++++++++ 3 files changed, 205 insertions(+), 110 deletions(-) create mode 100644 run_tests.go create mode 100644 std_test_pkg_exclusions diff --git a/.circleci/config.yml b/.circleci/config.yml index 1fbc65a56..7966dc43f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,6 +32,7 @@ jobs: cd node-syscall && node-gyp rebuild && mkdir -p $HOME/.node_libraries/ && cp build/Release/syscall.node $HOME/.node_libraries/syscall.node - run: | + go get myitcv.io/cmd/concsh go get -t ./... go install github.com/gopherjs/gopherjs @@ -43,115 +44,6 @@ jobs: for d in */; do echo $d; done | grep -v tests/ | grep -v third_party/ | xargs go tool vet # All subdirectories except "tests", "third_party". diff -u <(echo -n) <(go list ./compiler/natives/src/...) # All those packages should have // +build js. gopherjs install -v net/http # Should build successfully (can't run tests, since only client is supported). - ulimit -s 10000 && gopherjs test --minify -v --short \ - github.com/gopherjs/gopherjs/tests \ - github.com/gopherjs/gopherjs/tests/main \ - github.com/gopherjs/gopherjs/js \ - archive/tar \ - archive/zip \ - bufio \ - bytes \ - compress/bzip2 \ - compress/flate \ - compress/gzip \ - compress/lzw \ - compress/zlib \ - container/heap \ - container/list \ - container/ring \ - crypto/aes \ - crypto/cipher \ - crypto/des \ - crypto/dsa \ - crypto/ecdsa \ - crypto/elliptic \ - crypto/hmac \ - crypto/md5 \ - crypto/rand \ - crypto/rc4 \ - crypto/rsa \ - crypto/sha1 \ - crypto/sha256 \ - crypto/sha512 \ - crypto/subtle \ - crypto/x509 \ - database/sql \ - database/sql/driver \ - debug/dwarf \ - debug/elf \ - debug/macho \ - debug/pe \ - encoding/ascii85 \ - encoding/asn1 \ - encoding/base32 \ - encoding/base64 \ - encoding/binary \ - encoding/csv \ - encoding/gob \ - encoding/hex \ - encoding/json \ - encoding/pem \ - encoding/xml \ - errors \ - expvar \ - flag \ - fmt \ - go/ast \ - go/constant \ - go/doc \ - go/format \ - go/parser \ - go/printer \ - go/scanner \ - go/token \ - hash/adler32 \ - hash/crc32 \ - hash/crc64 \ - hash/fnv \ - html \ - html/template \ - image \ - image/color \ - image/draw \ - image/gif \ - image/jpeg \ - image/png \ - index/suffixarray \ - io \ - io/ioutil \ - math \ - math/big \ - math/bits \ - math/cmplx \ - math/rand \ - mime \ - mime/multipart \ - mime/quotedprintable \ - net/http/cookiejar \ - net/http/fcgi \ - net/mail \ - net/rpc/jsonrpc \ - net/textproto \ - net/url \ - os/user \ - path \ - path/filepath \ - reflect \ - regexp \ - regexp/syntax \ - sort \ - strconv \ - strings \ - sync \ - sync/atomic \ - testing/quick \ - text/scanner \ - text/tabwriter \ - text/template \ - text/template/parse \ - time \ - unicode \ - unicode/utf16 \ - unicode/utf8 + ulimit -s 10000 && go run run_tests.go -debug -p 2 go test -v -race ./... gopherjs test -v fmt # No minification should work. diff --git a/run_tests.go b/run_tests.go new file mode 100644 index 000000000..e74d4ebb6 --- /dev/null +++ b/run_tests.go @@ -0,0 +1,123 @@ +// +build ignore + +package main + +import ( + "bufio" + "flag" + "fmt" + "go/build" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" +) + +const ( + self = "github.com/gopherjs/gopherjs" +) + +var ( + fDebug = flag.Bool("debug", false, "include debug output") + fParallelism = flag.Int("p", runtime.NumCPU(), "number of pkgs to test in parallel") +) + +func main() { + flag.Parse() + + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "Failed to run tests: %v\n", err) + os.Exit(1) + } +} + +func run() error { + + bpkg, err := build.Import(self, ".", build.FindOnly) + if err != nil { + return fmt.Errorf("failed to get directory for import %v: %v", self, err) + } + + exPath := filepath.Join(bpkg.Dir, "std_test_pkg_exclusions") + + exFi, err := os.Open(exPath) + if err != nil { + return fmt.Errorf("error opening %v: %v", exPath, err) + } + + excls := make(map[string]bool) + + { + sc := bufio.NewScanner(exFi) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + + if strings.HasPrefix(line, "#") { + continue + } + + excls[line] = true + } + if err := sc.Err(); err != nil { + return fmt.Errorf("failed to line scan %v: %v", exPath, err) + } + } + + cmd := exec.Command("go", "list", "std") + stdListOut, err := cmd.Output() + if err != nil { + return fmt.Errorf("failed to %v: %v", strings.Join(cmd.Args, " "), err) + } + + tests := []string{ + "github.com/gopherjs/gopherjs/tests", + "github.com/gopherjs/gopherjs/tests/main", + "github.com/gopherjs/gopherjs/js", + } + + { + sc := bufio.NewScanner(strings.NewReader(string(stdListOut))) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + + if strings.HasPrefix(line, "#") { + continue + } + + if !excls[line] { + tests = append(tests, line) + } + } + if err := sc.Err(); err != nil { + return fmt.Errorf("failed to line scan %q: %v", strings.Join(cmd.Args, " "), err) + } + } + + var cmds []string + + for _, t := range tests { + cmds = append(cmds, fmt.Sprintf("gopherjs test -m %v\n", t)) + } + + p := *fParallelism + + debugf("running tests with parallelism %v\n", p) + + testCmd := exec.Command("concsh", "-conc", fmt.Sprintf("%v", p)) + testCmd.Stdin = strings.NewReader(strings.Join(cmds, "")) + testCmd.Stdout = os.Stdout + testCmd.Stderr = os.Stderr + + if err := testCmd.Run(); err != nil { + return fmt.Errorf("test process exited with an error: %v", err) + } + + return nil +} + +func debugf(format string, args ...interface{}) { + if *fDebug { + fmt.Fprintf(os.Stderr, format, args...) + } +} diff --git a/std_test_pkg_exclusions b/std_test_pkg_exclusions new file mode 100644 index 000000000..6a4b9d143 --- /dev/null +++ b/std_test_pkg_exclusions @@ -0,0 +1,80 @@ +# these are go list std packages we want to exclude from our tests +context +crypto +crypto/internal/cipherhw +crypto/tls +crypto/x509/pkix +debug/gosym +debug/plan9obj +encoding +go/build +go/importer +go/internal/gccgoimporter +go/internal/gcimporter +go/internal/srcimporter +go/types +hash +image/color/palette +image/internal/imageutil +internal/cpu +internal/nettrace +internal/poll +internal/race +internal/singleflight +internal/syscall/unix +internal/syscall/windows +internal/syscall/windows/registry +internal/syscall/windows/sysdll +internal/testenv +internal/testlog +internal/trace +log +log/syslog +net +net/http +net/http/cgi +net/http/httptest +net/http/httptrace +net/http/httputil +net/http/internal +net/http/pprof +net/internal/socktest +net/rpc +net/smtp +os +os/exec +os/signal +os/signal/internal/pty +plugin +runtime +runtime/cgo +runtime/debug +runtime/internal/atomic +runtime/internal/sys +runtime/pprof +runtime/pprof/internal/profile +runtime/race +runtime/trace +syscall +testing +testing/internal/testdeps +testing/iotest +unsafe +vendor/golang_org/x/crypto/chacha20poly1305 +vendor/golang_org/x/crypto/chacha20poly1305/internal/chacha20 +vendor/golang_org/x/crypto/cryptobyte +vendor/golang_org/x/crypto/cryptobyte/asn1 +vendor/golang_org/x/crypto/curve25519 +vendor/golang_org/x/crypto/poly1305 +vendor/golang_org/x/net/http2/hpack +vendor/golang_org/x/net/idna +vendor/golang_org/x/net/internal/nettest +vendor/golang_org/x/net/lex/httplex +vendor/golang_org/x/net/nettest +vendor/golang_org/x/net/proxy +vendor/golang_org/x/text/secure +vendor/golang_org/x/text/secure/bidirule +vendor/golang_org/x/text/transform +vendor/golang_org/x/text/unicode +vendor/golang_org/x/text/unicode/bidi +vendor/golang_org/x/text/unicode/norm