wip

coder · mafredri · Apr 3, 2023 · Mar 28, 2023 · Mar 28, 2023 · Mar 29, 2023
commit 004ca7422149e260b568a3f0d89bfa8c20b8855e
diff --git a/scripts/ci-report/main.go b/scripts/ci-report/main.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bufio"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -73,6 +74,8 @@ func parseCIReport(report GotestsumReport) (CIReport, error) {
 	testSkip := map[string]bool{}
 	testOutput := map[string]string{}
 	testSortedByName := []string{}
+	timeouts := map[string]string{}
+	timeoutRunningTests := map[string]bool{}
 	for i, e := range report {
 		switch e.Action {
 		// A package/test may fail or pass.
@@ -95,6 +98,11 @@ func parseCIReport(report GotestsumReport) (CIReport, error) {
 
 		// Gather all output (deleted when irrelevant).
 		case Output:
+			name := e.Package + "." + e.Test // May be pkg.Test or pkg.
+			if _, ok := timeouts[name]; ok || strings.HasPrefix(e.Output, "panic: test timed out") {
+				timeouts[name] += e.Output
+				continue
+			}
 			if e.Test != "" {
 				name := e.Package + "." + e.Test
 				testOutput[name] += e.Output
@@ -124,18 +132,49 @@ func parseCIReport(report GotestsumReport) (CIReport, error) {
 		}
 	}
 
+	// Normalize timeout from "pkg." or "pkg.Test" to "pkg".
+	timeoutsNorm := make(map[string]string)
+	for k, v := range timeouts {
+		names := strings.SplitN(k, ".", 2)
+		pkg := names[0]
+		if _, ok := timeoutsNorm[pkg]; ok {
+			panic("multiple timeouts for package: " + pkg)
+		}
+		timeoutsNorm[pkg] = v
+
+		// Mark all running tests as timed out.
+		// panic: test timed out after 2s\nrunning tests:\n\tTestAgent_Session_TTY_Hushlogin (0s)\n\n ...
+		parts := strings.SplitN(v, "\n", 3)
+		if len(parts) == 3 && strings.HasPrefix(parts[1], "running tests:") {
+			s := bufio.NewScanner(strings.NewReader(parts[2]))
+			for s.Scan() {
+				name := s.Text()
+				if !strings.HasPrefix(name, "\tTest") {
+					break
+				}
+				name = strings.TrimPrefix(name, "\t")
+				name = strings.SplitN(name, " ", 2)[0]
+				timeoutRunningTests[pkg+"."+name] = true
+			}
+		}
+	}
+	timeouts = timeoutsNorm
+
 	sortAZ := func(a, b string) bool { return a < b }
 	slices.SortFunc(packagesSortedByName, sortAZ)
 	slices.SortFunc(testSortedByName, sortAZ)
 
 	var rep CIReport
 
 	for _, pkg := range packagesSortedByName {
+		output, timeout := timeouts[pkg]
 		rep.Packages = append(rep.Packages, PackageReport{
 			Name:      pkg,
 			Time:      packageTimes[pkg],
 			Skip:      packageSkip[pkg],
 			Fail:      packageFail[pkg] > 0,
+			Timeout:   timeout,
+			Output:    output,
 			NumFailed: packageFail[pkg],
 		})
 	}
@@ -150,6 +189,7 @@ func parseCIReport(report GotestsumReport) (CIReport, error) {
 			Time:    testTimes[test],
 			Skip:    skip,
 			Fail:    fail,
+			Timeout: timeoutRunningTests[test],
 			Output:  out,
 		})
 	}
@@ -178,6 +218,8 @@ type PackageReport struct {
 	Skip      bool    `json:"skip,omitempty"`
 	Fail      bool    `json:"fail,omitempty"`
 	NumFailed int     `json:"num_failed,omitempty"`
+	Timeout   bool    `json:"timeout,omitempty"`
+	Output    string  `json:"output,omitempty"` // Output present e.g. for timeout.
 }
 
 type TestReport struct {
@@ -186,6 +228,7 @@ type TestReport struct {
 	Time    float64 `json:"time"`
 	Skip    bool    `json:"skip,omitempty"`
 	Fail    bool    `json:"fail,omitempty"`
+	Timeout bool    `json:"timeout,omitempty"`
 	Output  string  `json:"output,omitempty"`
 }
 

diff --git a/scripts/ci-report/main_test.go b/scripts/ci-report/main_test.go
@@ -17,38 +17,50 @@ var updateGoldenFiles = flag.Bool("update", false, "update .golden files")
 func TestOutputMatchesGoldenFile(t *testing.T) {
 	t.Parallel()
 
-	// Sample created via:
-	//	gotestsum --jsonfile ./scripts/ci-report/testdata/gotests.json.sample \
-	//	-- \
-	//	./agent ./cli ./cli/cliui \
-	//	-count=1 \
-	//	-timeout=5m \
-	//	-parallel=24 \
-	//	-run='^(TestServer|TestAgent_Session|TestGitAuth$|TestPrompt$)'
-	goTests, err := parseGoTestJSON(filepath.Join("testdata", "gotests.json.sample"))
-	if err != nil {
-		t.Fatalf("error parsing gotestsum report: %v", err)
-	}
+	for _, name := range []string{
+		// Sample created via:
+		//	gotestsum --jsonfile ./scripts/ci-report/testdata/gotests.json.sample -- \
+		//	./agent ./cli ./cli/cliui \
+		//	-count=1 \
+		//	-timeout=5m \
+		//	-parallel=24 \
+		//	-run='^(TestServer|TestAgent_Session|TestGitAuth$|TestPrompt$)'
+		filepath.Join("testdata", "gotests.json.sample"),
+		// Sample created via:
+		//	gotestsum --jsonfile ./scripts/ci-report/testdata/gotests-timeout.json.sample -- \
+		//	./agent -run='^TestAgent_Session' -count=1 -timeout=5m -parallel=24 -timeout=2s
+		filepath.Join("testdata", "gotests-timeout.json.sample"),
+	} {
+		name := name
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
 
-	rep, err := parseCIReport(goTests)
-	if err != nil {
-		t.Fatalf("error parsing ci report: %v", err)
-	}
+			goTests, err := parseGoTestJSON(name)
+			if err != nil {
+				t.Fatalf("error parsing gotestsum report: %v", err)
+			}
 
-	var b bytes.Buffer
-	err = printCIReport(&b, rep)
-	if err != nil {
-		t.Fatalf("error printing report: %v", err)
-	}
+			rep, err := parseCIReport(goTests)
+			if err != nil {
+				t.Fatalf("error parsing ci report: %v", err)
+			}
 
-	goldenFile := filepath.Join("testdata", "ci-report.golden")
-	got := b.Bytes()
-	if updateGoldenFile(t, goldenFile, got) {
-		return
-	}
+			var b bytes.Buffer
+			err = printCIReport(&b, rep)
+			if err != nil {
+				t.Fatalf("error printing report: %v", err)
+			}
+
+			goldenFile := filepath.Join("testdata", "ci-report_"+filepath.Base(name)+".golden")
+			got := b.Bytes()
+			if updateGoldenFile(t, goldenFile, got) {
+				return
+			}
 
-	want := readGoldenFile(t, goldenFile)
-	require.Equal(t, string(want), string(got))
+			want := readGoldenFile(t, goldenFile)
+			require.Equal(t, string(want), string(got))
+		})
+	}
 }
 
 func readGoldenFile(t *testing.T, name string) []byte {

diff --git a/scripts/ci-report/testdata/ci-report_gotests-timeout.json.sample.golden b/scripts/ci-report/testdata/ci-report_gotests-timeout.json.sample.golden
@@ -0,0 +1,43 @@
+{
+  "packages": [
+    {
+      "name": "agent",
+      "time": 2.045
+    }
+  ],
+  "tests": [
+    {
+      "package": "agent",
+      "name": "TestAgent_SessionExec",
+      "time": 0,
+      "fail": true,
+      "output": "=== RUN   TestAgent_SessionExec\n=== PAUSE TestAgent_SessionExec\n"
+    },
+    {
+      "package": "agent",
+      "name": "TestAgent_SessionTTYExitCode",
+      "time": 0,
+      "fail": true,
+      "output": "=== RUN   TestAgent_SessionTTYExitCode\n=== PAUSE TestAgent_SessionTTYExitCode\n"
+    },
+    {
+      "package": "agent",
+      "name": "TestAgent_SessionTTYShell",
+      "time": 0,
+      "fail": true,
+      "output": "=== RUN   TestAgent_SessionTTYShell\n=== PAUSE TestAgent_SessionTTYShell\n"
+    },
+    {
+      "package": "agent",
+      "name": "TestAgent_Session_TTY_Hushlogin",
+      "time": 0,
+      "fail": true,
+      "output": "=== RUN   TestAgent_Session_TTY_Hushlogin\npanic: test timed out after 2s\nrunning tests:\n\tTestAgent_Session_TTY_Hushlogin (0s)\n\ngoroutine 411 [running]:\ntesting.(*M).startAlarm.func1()\n\t/usr/local/go/src/testing/testing.go:2241 +0x3b9\ncreated by time.goFunc\n\t/usr/local/go/src/time/sleep.go:176 +0x32\n\ngoroutine 1 [chan receive]:\ntesting.(*T).Run(0xc0004e1040, {0x16a5e92?, 0x535fa5?}, 0x17462c0)\n\t/usr/local/go/src/testing/testing.go:1630 +0x405\ntesting.runTests.func1(0x236db60?)\n\t/usr/local/go/src/testing/testing.go:2036 +0x45\ntesting.tRunner(0xc0004e1040, 0xc000589bb8)\n\t/usr/local/go/src/testing/testing.go:1576 +0x10b\ntesting.runTests(0xc000341a40?, {0x235c580, 0x21, 0x21}, {0x4182d0?, 0xc000589c78?, 0x236cb40?})\n\t/usr/local/go/src/testing/testing.go:2034 +0x489\ntesting.(*M).Run(0xc000341a40)\n\t/usr/local/go/src/testing/testing.go:1906 +0x63a\ngo.uber.org/goleak.VerifyTestMain({0x18f5540?, 0xc000341a40?}, {0x0, 0x0, 0x0})\n\t/home/mafredri/.local/go/pkg/mod/go.uber.org/goleak@v1.2.1/testmain.go:53 +0x6b\ngithub.com/coder/coder/agent_test.TestMain(...)\n\t/home/mafredri/src/coder/coder/agent/agent_test.go:53\nmain.main()\n\t_testmain.go:115 +0x1e5\n\ngoroutine 9 [chan receive]:\ntesting.(*T).Parallel(0xc0004e11e0)\n\t/usr/local/go/src/testing/testing.go:1384 +0x225\ngithub.com/coder/coder/agent_test.TestAgent_SessionExec(0x0?)\n\t/home/mafredri/src/coder/coder/agent/agent_test.go:188 +0x33\ntesting.tRunner(0xc0004e11e0, 0x1746298)\n\t/usr/local/go/src/testing/testing.go:1576 +0x10b\ncreated by testing.(*T).Run\n\t/usr/local/go/src/testing/testing.go:1629 +0x3ea\n\ngoroutine 10 [chan receive]:\ntesting.(*T).Parallel(0xc0004e1520)\n\t/usr/local/go/src/testing/testing.go:1384 +0x225\ngithub.com/coder/coder/agent_test.TestAgent_SessionTTYShell(0xc0004e1520)\n\t/home/mafredri/src/coder/coder/agent/agent_test.go:213 +0x36\ntesting.tRunner(0xc0004e1520, 0x17462a8)\n\t/usr/local/go/src/testing/testing.go:1576 +0x10b\ncreated by testing.(*T).Run\n\t/usr/local/go/src/testing/testing.go:1629 +0x3ea\n\ngoroutine 11 [chan receive]:\ntesting.(*T).Parallel(0xc0004e1860)\n\t/usr/local/go/src/testing/testing.go:1384 +0x225\ngithub.com/coder/coder/agent_test.TestAgent_SessionTTYExitCode(0xc0004e1520?)\n\t/home/mafredri/src/coder/coder/agent/agent_test.go:244 +0x36\ntesting.tRunner(0xc0004e1860, 0x17462a0)\n\t/usr/local/go/src/testing/testing.go:1576 +0x10b\ncreated by testing.(*T).Run\n\t/usr/local/go/src/testing/testing.go:1629 +0x3ea\n\ngoroutine 408 [runnable]:\nmath/big.nat.montgomery({0xc004aa4500?, 0x10?, 0x26?}, {0xc004aa4280?, 0x10?, 0x26?}, {0xc004aa4280?, 0x10?, 0x26?}, {0xc000732820, ...}, ...)\n\t/usr/local/go/src/math/big/nat.go:216 +0x565\nmath/big.nat.expNNMontgomery({0xc004aa4280, 0xc0003c2e70?, 0x26}, {0xc004a9adc0?, 0x21?, 0x24?}, {0xc004a9ac80, 0x10, 0x24?}, {0xc000732820, ...})\n\t/usr/local/go/src/math/big/nat.go:1271 +0xb1c\nmath/big.nat.expNN({0xc004aa4280?, 0x14?, 0x22c2900?}, {0xc004a9adc0?, 0x10, 0x14}, {0xc004a9ac80?, 0x10, 0x14?}, {0xc000732820, ...}, ...)\n\t/usr/local/go/src/math/big/nat.go:996 +0x3b1\nmath/big.nat.probablyPrimeMillerRabin({0xc000732820?, 0x10, 0x14}, 0x15, 0x1)\n\t/usr/local/go/src/math/big/prime.go:106 +0x5b8\nmath/big.(*Int).ProbablyPrime(0xc0047208c0, 0x14)\n\t/usr/local/go/src/math/big/prime.go:78 +0x225\ncrypto/rand.Prime({0x18f04c0, 0xc00007e020}, 0x400)\n\t/usr/local/go/src/crypto/rand/util.go:55 +0x1e5\ncrypto/rsa.GenerateMultiPrimeKey({0x18f04c0, 0xc00007e020}, 0x2, 0x800)\n\t/usr/local/go/src/crypto/rsa/rsa.go:369 +0x745\ncrypto/rsa.GenerateKey(...)\n\t/usr/local/go/src/crypto/rsa/rsa.go:264\ngithub.com/coder/coder/agent.(*agent).init(0xc00485eea0, {0x1902c20?, 0xc00485d770})\n\t/home/mafredri/src/coder/coder/agent/agent.go:810 +0x6c\ngithub.com/coder/coder/agent.New({{0x190cbc0, 0xc0005b7710}, {0x166d829, 0x4}, {0x166d829, 0x4}, 0x17461d8, {0x1907c90, 0xc000278280}, 0x45d964b800, ...})\n\t/home/mafredri/src/coder/coder/agent/agent.go:134 +0x549\ngithub.com/coder/coder/agent_test.setupAgent(0xc00485eb60, {0x0, {0x0, 0x0}, {0x0, 0x0, 0x0}, 0xc0005b8da0, 0x0, {0x0, ...}, ...}, ...)\n\t/home/mafredri/src/coder/coder/agent/agent_test.go:1568 +0x63e\ngithub.com/coder/coder/agent_test.setupSSHSession(0xc00485eb60, {0x0, {0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0, {0x0, ...}, ...})\n\t/home/mafredri/src/coder/coder/agent/agent_test.go:1524 +0xc5\ngithub.com/coder/coder/agent_test.TestAgent_Session_TTY_Hushlogin(0xc00485eb60)\n\t/home/mafredri/src/coder/coder/agent/agent_test.go:330 +0x2fa\ntesting.tRunner(0xc00485eb60, 0x17462c0)\n\t/usr/local/go/src/testing/testing.go:1576 +0x10b\ncreated by testing.(*T).Run\n\t/usr/local/go/src/testing/testing.go:1629 +0x3ea\n\ngoroutine 409 [IO wait]:\ninternal/poll.runtime_pollWait(0x7f5230766628, 0x72)\n\t/usr/local/go/src/runtime/netpoll.go:306 +0x89\ninternal/poll.(*pollDesc).wait(0xc00475bf80?, 0xc0005ec5e2?, 0x0)\n\t/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/local/go/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Accept(0xc00475bf80)\n\t/usr/local/go/src/internal/poll/fd_unix.go:614 +0x2bd\nnet.(*netFD).accept(0xc00475bf80)\n\t/usr/local/go/src/net/fd_unix.go:172 +0x35\nnet.(*TCPListener).accept(0xc00486ecd8)\n\t/usr/local/go/src/net/tcpsock_posix.go:148 +0x25\nnet.(*TCPListener).Accept(0xc00486ecd8)\n\t/usr/local/go/src/net/tcpsock.go:297 +0x3d\ncrypto/tls.(*listener).Accept(0xc00486ef18)\n\t/usr/local/go/src/crypto/tls/tls.go:66 +0x2d\nnet/http.(*Server).Serve(0xc00029da40, {0x18fefa0, 0xc00486ef18})\n\t/usr/local/go/src/net/http/server.go:3059 +0x385\nnet/http/httptest.(*Server).goServe.func1()\n\t/usr/local/go/src/net/http/httptest/server.go:310 +0x6a\ncreated by net/http/httptest.(*Server).goServe\n\t/usr/local/go/src/net/http/httptest/server.go:308 +0x6a\n\ngoroutine 410 [IO wait]:\ninternal/poll.runtime_pollWait(0x7f5230765908, 0x72)\n\t/usr/local/go/src/runtime/netpoll.go:306 +0x89\ninternal/poll.(*pollDesc).wait(0xc00043a300?, 0xc004880000?, 0x0)\n\t/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/local/go/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).ReadFromInet4(0xc00043a300, {0xc004880000, 0x10000, 0x10000}, 0x0?)\n\t/usr/local/go/src/internal/poll/fd_unix.go:250 +0x24f\nnet.(*netFD).readFromInet4(0xc00043a300, {0xc004880000?, 0x0?, 0x0?}, 0x0?)\n\t/usr/local/go/src/net/fd_posix.go:66 +0x29\nnet.(*UDPConn).readFrom(0x30?, {0xc004880000?, 0xc0005b7770?, 0x0?}, 0xc0005b7770)\n\t/usr/local/go/src/net/udpsock_posix.go:52 +0x1b8\nnet.(*UDPConn).readFromUDP(0xc000015a08, {0xc004880000?, 0x4102c7?, 0x10000?}, 0x13e45e0?)\n\t/usr/local/go/src/net/udpsock.go:149 +0x31\nnet.(*UDPConn).ReadFrom(0x59a?, {0xc004880000, 0x10000, 0x10000})\n\t/usr/local/go/src/net/udpsock.go:158 +0x50\ntailscale.com/net/stun/stuntest.runSTUN({0x1911ec0, 0xc00485eb60}, {0x1907f60, 0xc000015a08}, 0xc00481baa0, 0x17462c0?)\n\t/home/mafredri/.local/go/pkg/mod/github.com/coder/tailscale@v1.1.1-0.20230327205451-058fa46a3723/net/stun/stuntest/stuntest.go:59 +0xc6\ncreated by tailscale.com/net/stun/stuntest.ServeWithPacketListener\n\t/home/mafredri/.local/go/pkg/mod/github.com/coder/tailscale@v1.1.1-0.20230327205451-058fa46a3723/net/stun/stuntest/stuntest.go:47 +0x26a\n"
+    },
+    {
+      "package": "agent",
+      "name": "TestAgent_Session_TTY_MOTD",
+      "time": 1.84
+    }
+  ]
+}
diff --git a/scripts/ci-report/testdata/ci-report.golden → ...data/ci-report_gotests.json.sample.golden b/scripts/ci-report/testdata/ci-report.golden → ...data/ci-report_gotests.json.sample.golden