Skip to content

Commit 2b5e02f

Browse files
aslilacaslilac
authored
refactor: improve e2e test reporting (#10304)
1 parent ab45627 commit 2b5e02f

File tree

3 files changed

+114
-50
lines changed

3 files changed

+114
-50
lines changed

‎site/.eslintrc.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,9 @@ rules:
9191
- allowSingleExtends: true
9292
"brace-style": "off"
9393
"curly": ["error", "all"]
94+
"eslint-comments/disable-enable-pair":
95+
- error
96+
- allowWholeFile: true
9497
"eslint-comments/require-description": "error"
9598
eqeqeq: error
9699
import/default: "off"

‎site/e2e/playwright.config.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ export const port = process.env.CODER_E2E_PORT
66
? Number(process.env.CODER_E2E_PORT)
77
: defaultPort;
88

9-
const coderMain = path.join(__dirname, "../../enterprise/cmd/coder/main.go");
9+
const coderMain = path.join(__dirname, "../../enterprise/cmd/coder");
1010

1111
export const STORAGE_STATE = path.join(__dirname, ".auth.json");
1212

‎site/e2e/reporter.ts

+110-49
Original file line numberDiff line numberDiff line change
@@ -1,85 +1,146 @@
1-
import fs from "fs";
1+
/* eslint-disable no-console -- Logging is sort of the whole point here */
2+
import * as fs from "fs/promises";
23
import type {
34
FullConfig,
45
Suite,
56
TestCase,
67
TestResult,
78
FullResult,
89
Reporter,
10+
TestError,
911
} from "@playwright/test/reporter";
1012
import axios from "axios";
13+
import type { Writable } from "stream";
1114

1215
class CoderReporter implements Reporter {
16+
config: FullConfig | null = null;
17+
testOutput = new Map<string, Array<[Writable, string]>>();
18+
passedCount = 0;
19+
failedTests: TestCase[] = [];
20+
timedOutTests: TestCase[] = [];
21+
1322
onBegin(config: FullConfig, suite: Suite) {
14-
// eslint-disable-next-line no-console -- Helpful for debugging
15-
console.log(`Starting the run with ${suite.allTests().length} tests`);
23+
this.config = config;
24+
console.log(`==> Running ${suite.allTests().length} tests`);
1625
}
1726

1827
onTestBegin(test: TestCase) {
19-
// eslint-disable-next-line no-console -- Helpful for debugging
20-
console.log(`Starting test ${test.title}`);
28+
this.testOutput.set(test.id, []);
29+
console.log(`==> Starting test ${test.title}`);
2130
}
2231

23-
onStdOut(chunk: string, test: TestCase, _: TestResult): void {
24-
// eslint-disable-next-line no-console -- Helpful for debugging
25-
console.log(
26-
`[stdout] [${test ? test.title : "unknown"}]: ${chunk.replace(
27-
/\n$/g,
28-
"",
29-
)}`,
30-
);
32+
onStdOut(chunk: string, test?: TestCase, _?: TestResult): void {
33+
if (!test) {
34+
for (const line of filteredServerLogLines(chunk)) {
35+
console.log(`[stdout] ${line}`);
36+
}
37+
return;
38+
}
39+
this.testOutput.get(test.id)!.push([process.stdout, chunk]);
3140
}
3241

33-
onStdErr(chunk: string, test: TestCase, _: TestResult): void {
34-
// eslint-disable-next-line no-console -- Helpful for debugging
35-
console.log(
36-
`[stderr] [${test ? test.title : "unknown"}]: ${chunk.replace(
37-
/\n$/g,
38-
"",
39-
)}`,
40-
);
42+
onStdErr(chunk: string, test?: TestCase, _?: TestResult): void {
43+
if (!test) {
44+
for (const line of filteredServerLogLines(chunk)) {
45+
console.error(`[stderr] ${line}`);
46+
}
47+
return;
48+
}
49+
this.testOutput.get(test.id)!.push([process.stderr, chunk]);
4150
}
4251

4352
async onTestEnd(test: TestCase, result: TestResult) {
44-
// eslint-disable-next-line no-console -- Helpful for debugging
45-
console.log(`Finished test ${test.title}: ${result.status}`);
53+
console.log(`==> Finished test ${test.title}: ${result.status}`);
54+
55+
if (result.status === "passed") {
56+
this.passedCount++;
57+
}
58+
59+
if (result.status === "failed") {
60+
this.failedTests.push(test);
61+
}
62+
63+
if (result.status === "timedOut") {
64+
this.timedOutTests.push(test);
65+
}
66+
67+
const fsTestTitle = test.title.replaceAll(" ", "-");
68+
const outputFile = `test-results/debug-pprof-goroutine-${fsTestTitle}.txt`;
69+
await exportDebugPprof(outputFile);
4670

4771
if (result.status !== "passed") {
48-
// eslint-disable-next-line no-console -- Helpful for debugging
49-
console.log("errors", result.errors, "attachments", result.attachments);
72+
console.log(`Data from pprof has been saved to ${outputFile}`);
73+
console.log("==> Output");
74+
const output = this.testOutput.get(test.id)!;
75+
for (const [target, chunk] of output) {
76+
target.write(`${chunk.replace(/\n$/g, "")}\n`);
77+
}
78+
79+
if (result.errors.length > 0) {
80+
console.log("==> Errors");
81+
for (const error of result.errors) {
82+
reportError(error);
83+
}
84+
}
85+
86+
if (result.attachments.length > 0) {
87+
console.log("==> Attachments");
88+
for (const attachment of result.attachments) {
89+
console.log(attachment);
90+
}
91+
}
5092
}
51-
await exportDebugPprof(test.title);
93+
this.testOutput.delete(test.id);
5294
}
5395

5496
onEnd(result: FullResult) {
55-
// eslint-disable-next-line no-console -- Helpful for debugging
56-
console.log(`Finished the run: ${result.status}`);
97+
console.log(`==> Tests ${result.status}`);
98+
console.log(`${this.passedCount} passed`);
99+
if (this.failedTests.length > 0) {
100+
console.log(`${this.failedTests.length} failed`);
101+
for (const test of this.failedTests) {
102+
console.log(` ${test.location.file} › ${test.title}`);
103+
}
104+
}
105+
if (this.timedOutTests.length > 0) {
106+
console.log(`${this.timedOutTests.length} timed out`);
107+
for (const test of this.timedOutTests) {
108+
console.log(` ${test.location.file} › ${test.title}`);
109+
}
110+
}
57111
}
58112
}
59113

60-
const exportDebugPprof = async (testName: string) => {
61-
const url = "http://127.0.0.1:6060/debug/pprof/goroutine?debug=1";
62-
const outputFile = `test-results/debug-pprof-goroutine-${testName}.txt`;
114+
const shouldPrintLine = (line: string) =>
115+
[" error=EOF", "coderd: audit_log"].every((noise) => !line.includes(noise));
63116

64-
await axios
65-
.get(url)
66-
.then((response) => {
67-
if (response.status !== 200) {
68-
throw new Error(`Error: Received status code ${response.status}`);
69-
}
117+
const filteredServerLogLines = (chunk: string): string[] =>
118+
chunk.trimEnd().split("\n").filter(shouldPrintLine);
70119

71-
fs.writeFile(outputFile, response.data, (err) => {
72-
if (err) {
73-
throw new Error(`Error writing to ${outputFile}: ${err.message}`);
74-
} else {
75-
// eslint-disable-next-line no-console -- Helpful for debugging
76-
console.log(`Data from ${url} has been saved to ${outputFile}`);
77-
}
78-
});
79-
})
80-
.catch((error) => {
81-
throw new Error(`Error: ${error.message}`);
82-
});
120+
const exportDebugPprof = async (outputFile: string) => {
121+
const response = await axios.get(
122+
"http://127.0.0.1:6060/debug/pprof/goroutine?debug=1",
123+
);
124+
if (response.status !== 200) {
125+
throw new Error(`Error: Received status code ${response.status}`);
126+
}
127+
128+
await fs.writeFile(outputFile, response.data);
129+
};
130+
131+
const reportError = (error: TestError) => {
132+
if (error.location) {
133+
console.log(`${error.location.file}:${error.location.line}:`);
134+
}
135+
if (error.snippet) {
136+
console.log(error.snippet);
137+
}
138+
139+
if (error.message) {
140+
console.log(error.message);
141+
} else {
142+
console.log(error);
143+
}
83144
};
84145

85146
// eslint-disable-next-line no-unused-vars -- Playwright config uses it

0 commit comments

Comments
 (0)