ci: Print go test stats

Fixes #6676
coder · mafredri · Apr 3, 2023 · Mar 28, 2023 · Mar 28, 2023 · Mar 29, 2023
commit 0aeb408017116f15ed9c6c879007566fda0591c9
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -302,7 +302,14 @@ jobs:
             echo "cover=false" >> $GITHUB_OUTPUT
           fi
 
-          gotestsum --junitfile="gotests.xml" --packages="./..." -- -parallel=8 -timeout=7m -short -failfast $COVERAGE_FLAGS
+          gotestsum --junitfile="gotests.xml" --jsonfile="gotests.json" --packages="./..." -- -parallel=8 -timeout=7m -short -failfast $COVERAGE_FLAGS
+
+      - name: Print test stats
+        if: success() || failure()
+        run: |
+          # Artifacts are not available after rerunning a job,
+          # so we need to print the test stats to the log.
+          go run ./scripts/ci-report/main.go gotests.json | tee gotests_stats.json
 
       - uses: actions/upload-artifact@v3
         if: success() || failure()
@@ -372,6 +379,13 @@ jobs:
         run: |
           make test-postgres
 
+      - name: Print test stats
+        if: success() || failure()
+        run: |
+          # Artifacts are not available after rerunning a job,
+          # so we need to print the test stats to the log.
+          go run ./scripts/ci-report/main.go gotests.json | tee gotests_stats.json
+
       - uses: actions/upload-artifact@v3
         if: success() || failure()
         with:

diff --git a/.gitignore b/.gitignore
@@ -6,7 +6,8 @@
 **/*.swp
 gotests.coverage
 gotests.xml
-gotestsum.json
+gotests_stats.json
+gotests.json
 node_modules/
 vendor/
 yarn-error.log

diff --git a/Makefile b/Makefile
@@ -596,6 +596,7 @@ test-postgres: test-clean test-postgres-docker
 	# more consistent execution.
 	DB=ci DB_FROM=$(shell go run scripts/migrate-ci/main.go) gotestsum \
 		--junitfile="gotests.xml" \
+		--jsonfile="gotests.json" \
 		--packages="./..." -- \
 		-covermode=atomic -coverprofile="gotests.coverage" -timeout=20m \
 		-parallel=4 \

diff --git a/scripts/ci-report/main.go b/scripts/ci-report/main.go
@@ -0,0 +1,188 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"time"
+
+	"golang.org/x/exp/slices"
+)
+
+func main() {
+	if len(os.Args) != 2 {
+		_, _ = fmt.Println("usage: ci-report <gotests.json>")
+		os.Exit(1)
+	}
+	name := os.Args[1]
+
+	f, err := os.Open(name)
+	if err != nil {
+		_, _ = fmt.Printf("error opening gotestsum json file: %v", err)
+		os.Exit(1)
+	}
+	defer f.Close()
+
+	dec := json.NewDecoder(f)
+	var report GotestsumReport
+	for {
+		var e GotestsumReportEntry
+		err = dec.Decode(&e)
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			_, _ = fmt.Printf("error decoding json: %v", err)
+			os.Exit(1)
+		}
+		e.Package = strings.TrimPrefix(e.Package, "github.com/coder/coder/")
+		report = append(report, e)
+	}
+
+	packagesSortedByName := []string{}
+	packageTimes := map[string]float64{}
+	packageFail := map[string]int{}
+	packageSkip := map[string]bool{}
+	testTimes := map[string]float64{}
+	testSkip := map[string]bool{}
+	testOutput := map[string]string{}
+	testSortedByName := []string{}
+	for i, e := range report {
+		switch e.Action {
+		// A package/test may fail or pass.
+		case Fail:
+			if e.Test == "" {
+				packageTimes[e.Package] = *e.Elapsed
+			} else {
+				packageFail[e.Package]++
+				name := e.Package + "." + e.Test
+				testTimes[name] = *e.Elapsed
+			}
+		case Pass:
+			if e.Test == "" {
+				packageTimes[e.Package] = *e.Elapsed
+			} else {
+				name := e.Package + "." + e.Test
+				delete(testOutput, name)
+				testTimes[name] = *e.Elapsed
+			}
+
+		// Gather all output (deleted when irrelevant).
+		case Output:
+			if e.Test != "" {
+				name := e.Package + "." + e.Test
+				testOutput[name] += e.Output
+			}
+
+		// Packages start, tests run and either may be skipped.
+		case Start:
+			packagesSortedByName = append(packagesSortedByName, e.Package)
+		case Run:
+			name := e.Package + "." + e.Test
+			testSortedByName = append(testSortedByName, name)
+		case Skip:
+			if e.Test == "" {
+				packageSkip[e.Package] = true
+			} else {
+				name := e.Package + "." + e.Test
+				testSkip[name] = true
+				delete(testOutput, name)
+			}
+
+		// Ignore.
+		case Cont:
+		case Pause:
+
+		default:
+			_, _ = fmt.Printf("unknown action: %v in entry %d (%v)", e.Action, i, e)
+			os.Exit(1)
+		}
+	}
+
+	sortAZ := func(a, b string) bool { return a < b }
+	slices.SortFunc(packagesSortedByName, sortAZ)
+	slices.SortFunc(testSortedByName, sortAZ)
+
+	var rep CIReport
+
+	for _, pkg := range packagesSortedByName {
+		rep.Packages = append(rep.Packages, PackageReport{
+			Name:      pkg,
+			Time:      packageTimes[pkg],
+			Skip:      packageSkip[pkg],
+			Fail:      packageFail[pkg] > 0,
+			NumFailed: packageFail[pkg],
+		})
+	}
+
+	for _, test := range testSortedByName {
+		names := strings.SplitN(test, ".", 2)
+		skip := testSkip[test]
+		out, fail := testOutput[test]
+		rep.Tests = append(rep.Tests, TestReport{
+			Package: names[0],
+			Name:    names[1],
+			Time:    testTimes[test],
+			Skip:    skip,
+			Fail:    fail,
+			Output:  out,
+		})
+	}
+
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	err = enc.Encode(rep)
+	if err != nil {
+		_, _ = fmt.Printf("error encoding json: %v", err)
+		os.Exit(1)
+	}
+}
+
+type CIReport struct {
+	Packages []PackageReport `json:"packages"`
+	Tests    []TestReport    `json:"tests"`
+}
+
+type PackageReport struct {
+	Name      string  `json:"name"`
+	Time      float64 `json:"time"`
+	Skip      bool    `json:"skip,omitempty"`
+	Fail      bool    `json:"fail,omitempty"`
+	NumFailed int     `json:"num_failed,omitempty"`
+}
+
+type TestReport struct {
+	Package string  `json:"package"`
+	Name    string  `json:"name"`
+	Time    float64 `json:"time"`
+	Skip    bool    `json:"skip,omitempty"`
+	Fail    bool    `json:"fail,omitempty"`
+	Output  string  `json:"output,omitempty"`
+}
+
+type GotestsumReport []GotestsumReportEntry
+
+type GotestsumReportEntry struct {
+	Time    time.Time `json:"Time"`
+	Action  Action    `json:"Action"`
+	Package string    `json:"Package"`
+	Test    string    `json:"Test,omitempty"`
+	Output  string    `json:"Output,omitempty"`
+	Elapsed *float64  `json:"Elapsed,omitempty"`
+}
+
+type Action string
+
+const (
+	Cont   Action = "cont"
+	Fail   Action = "fail"
+	Output Action = "output"
+	Pass   Action = "pass"
+	Pause  Action = "pause"
+	Run    Action = "run"
+	Skip   Action = "skip"
+	Start  Action = "start"
+)