From 54c4b2d18d1e25338631de0be0a85503c6ab958d Mon Sep 17 00:00:00 2001 From: McKayla Washburn Date: Mon, 16 Oct 2023 18:13:18 +0000 Subject: [PATCH 1/7] refactor: better e2e test reporting --- site/e2e/reporter.ts | 79 +++++++++++++++++++++++++++++++------------- 1 file changed, 56 insertions(+), 23 deletions(-) diff --git a/site/e2e/reporter.ts b/site/e2e/reporter.ts index 1cb5e34c6619a..ce78540758e7c 100644 --- a/site/e2e/reporter.ts +++ b/site/e2e/reporter.ts @@ -1,4 +1,4 @@ -import fs from "fs"; +import * as fs from "fs"; import type { FullConfig, Suite, @@ -8,52 +8,85 @@ import type { Reporter, } from "@playwright/test/reporter"; import axios from "axios"; +import type { Writable } from "stream"; + +const testOutput = new Map>(); class CoderReporter implements Reporter { onBegin(config: FullConfig, suite: Suite) { // eslint-disable-next-line no-console -- Helpful for debugging - console.log(`Starting the run with ${suite.allTests().length} tests`); + console.log(`==> Running ${suite.allTests().length} tests`); } onTestBegin(test: TestCase) { + testOutput.set(test.id, []); // eslint-disable-next-line no-console -- Helpful for debugging - console.log(`Starting test ${test.title}`); + console.log(`==> Starting test ${test.title}`); } - onStdOut(chunk: string, test: TestCase, _: TestResult): void { - // eslint-disable-next-line no-console -- Helpful for debugging - console.log( - `[stdout] [${test ? test.title : "unknown"}]: ${chunk.replace( - /\n$/g, - "", - )}`, - ); + onStdOut(chunk: string, test?: TestCase, _?: TestResult): void { + if (!test) { + // console.log(`[stdout] [unknown] ${chunk.replace(/\n$/g, "")}`); + return; + } + testOutput.get(test.id)!.push([process.stdout, chunk]); } - onStdErr(chunk: string, test: TestCase, _: TestResult): void { - // eslint-disable-next-line no-console -- Helpful for debugging - console.log( - `[stderr] [${test ? test.title : "unknown"}]: ${chunk.replace( - /\n$/g, - "", - )}`, - ); + onStdErr(chunk: string, test?: TestCase, _?: TestResult): void { + if (!test) { + // console.error(`[stderr] [unknown] ${chunk.replace(/\n$/g, "")}`); + return; + } + testOutput.get(test.id)!.push([process.stderr, chunk]); } async onTestEnd(test: TestCase, result: TestResult) { // eslint-disable-next-line no-console -- Helpful for debugging - console.log(`Finished test ${test.title}: ${result.status}`); + console.log(`==> Finished test ${test.title}: ${result.status}`); if (result.status !== "passed") { - // eslint-disable-next-line no-console -- Helpful for debugging - console.log("errors", result.errors, "attachments", result.attachments); + // eslint-disable-next-line no-console -- Debugging output + console.log("==> Output"); + const output = testOutput.get(test.id)!; + for (const [target, chunk] of output) { + target.write(`${chunk.replace(/\n$/g, "")}\n`); + } + testOutput.delete(test.id); + + if (result.errors.length > 0) { + // eslint-disable-next-line no-console -- Debugging output + console.log("==> Errors"); + for (const error of result.errors) { + if (error.location) { + console.log(`${error.location.file}:${error.location.line}:`); + } + if (error.snippet) { + console.log(error.snippet); + } + + if (error.message) { + console.log(error.message); + } else { + console.log(error); + } + } + } + + if (result.attachments.length > 0) { + // eslint-disable-next-line no-console -- Debugging output + console.log("==> Attachments"); + for (const attachment of result.attachments) { + // eslint-disable-next-line no-console -- Debugging output + console.log(attachment); + } + } } await exportDebugPprof(test.title); } onEnd(result: FullResult) { // eslint-disable-next-line no-console -- Helpful for debugging - console.log(`Finished the run: ${result.status}`); + console.log(`==> Tests ${result.status}`); } } From 1096c2f119967765fdeeed29e56c044da84d62ea Mon Sep 17 00:00:00 2001 From: McKayla Washburn Date: Mon, 16 Oct 2023 18:29:21 +0000 Subject: [PATCH 2/7] =?UTF-8?q?=F0=9F=A7=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- site/e2e/reporter.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/site/e2e/reporter.ts b/site/e2e/reporter.ts index ce78540758e7c..2008584ad2070 100644 --- a/site/e2e/reporter.ts +++ b/site/e2e/reporter.ts @@ -51,22 +51,25 @@ class CoderReporter implements Reporter { for (const [target, chunk] of output) { target.write(`${chunk.replace(/\n$/g, "")}\n`); } - testOutput.delete(test.id); if (result.errors.length > 0) { // eslint-disable-next-line no-console -- Debugging output console.log("==> Errors"); for (const error of result.errors) { if (error.location) { + // eslint-disable-next-line no-console -- Debugging output console.log(`${error.location.file}:${error.location.line}:`); } if (error.snippet) { + // eslint-disable-next-line no-console -- Debugging output console.log(error.snippet); } if (error.message) { + // eslint-disable-next-line no-console -- Debugging output console.log(error.message); } else { + // eslint-disable-next-line no-console -- Debugging output console.log(error); } } @@ -81,6 +84,7 @@ class CoderReporter implements Reporter { } } } + testOutput.delete(test.id); await exportDebugPprof(test.title); } From 011dc55c710dfcfad3c1b76882f0d090f1d4812a Mon Sep 17 00:00:00 2001 From: McKayla Washburn Date: Mon, 16 Oct 2023 19:24:35 +0000 Subject: [PATCH 3/7] report specific test failure names --- site/e2e/playwright.config.ts | 1 + site/e2e/reporter.ts | 107 ++++++++++++++++++++++++---------- 2 files changed, 76 insertions(+), 32 deletions(-) diff --git a/site/e2e/playwright.config.ts b/site/e2e/playwright.config.ts index 792944f26dde9..7782ad5245e87 100644 --- a/site/e2e/playwright.config.ts +++ b/site/e2e/playwright.config.ts @@ -15,6 +15,7 @@ const localURL = (port: number, path: string): string => { }; export default defineConfig({ + preserveOutput: "failures-only", projects: [ { name: "setup", diff --git a/site/e2e/reporter.ts b/site/e2e/reporter.ts index 2008584ad2070..bc6fd32d96cdc 100644 --- a/site/e2e/reporter.ts +++ b/site/e2e/reporter.ts @@ -6,72 +6,82 @@ import type { TestResult, FullResult, Reporter, + TestError, } from "@playwright/test/reporter"; import axios from "axios"; import type { Writable } from "stream"; -const testOutput = new Map>(); - class CoderReporter implements Reporter { + config: FullConfig | null = null; + testOutput = new Map>(); + passedCount = 0; + failedTests: TestCase[] = []; + timedOutTests: TestCase[] = []; + onBegin(config: FullConfig, suite: Suite) { + this.config = config; // eslint-disable-next-line no-console -- Helpful for debugging console.log(`==> Running ${suite.allTests().length} tests`); } onTestBegin(test: TestCase) { - testOutput.set(test.id, []); + this.testOutput.set(test.id, []); // eslint-disable-next-line no-console -- Helpful for debugging console.log(`==> Starting test ${test.title}`); } onStdOut(chunk: string, test?: TestCase, _?: TestResult): void { if (!test) { - // console.log(`[stdout] [unknown] ${chunk.replace(/\n$/g, "")}`); + const preserve = this.config?.preserveOutput === "always"; + if (preserve) { + console.log(`[stdout] ${chunk.replace(/\n$/g, "")}`); + } return; } - testOutput.get(test.id)!.push([process.stdout, chunk]); + this.testOutput.get(test.id)!.push([process.stdout, chunk]); } onStdErr(chunk: string, test?: TestCase, _?: TestResult): void { if (!test) { - // console.error(`[stderr] [unknown] ${chunk.replace(/\n$/g, "")}`); + const preserve = this.config?.preserveOutput === "always"; + if (preserve) { + console.error(`[stderr] ${chunk.replace(/\n$/g, "")}`); + } return; } - testOutput.get(test.id)!.push([process.stderr, chunk]); + this.testOutput.get(test.id)!.push([process.stderr, chunk]); } async onTestEnd(test: TestCase, result: TestResult) { - // eslint-disable-next-line no-console -- Helpful for debugging - console.log(`==> Finished test ${test.title}: ${result.status}`); + console.log(`==> Finished test ${test.title}: ${result.status}`); // eslint-disable-line no-console -- Helpful for debugging + + if (result.status === "passed") { + this.passedCount++; + } + + if (result.status === "failed") { + this.failedTests.push(test); + } - if (result.status !== "passed") { - // eslint-disable-next-line no-console -- Debugging output - console.log("==> Output"); - const output = testOutput.get(test.id)!; + if (result.status === "timedOut") { + this.timedOutTests.push(test); + } + + const preserve = this.config?.preserveOutput; + const logOutput = + preserve === "always" || + (result.status !== "passed" && preserve !== "never"); + if (logOutput) { + console.log("==> Output"); // eslint-disable-line no-console -- Debugging output + const output = this.testOutput.get(test.id)!; for (const [target, chunk] of output) { target.write(`${chunk.replace(/\n$/g, "")}\n`); } if (result.errors.length > 0) { - // eslint-disable-next-line no-console -- Debugging output - console.log("==> Errors"); + console.log("==> Errors"); // eslint-disable-line no-console -- Debugging output for (const error of result.errors) { - if (error.location) { - // eslint-disable-next-line no-console -- Debugging output - console.log(`${error.location.file}:${error.location.line}:`); - } - if (error.snippet) { - // eslint-disable-next-line no-console -- Debugging output - console.log(error.snippet); - } - - if (error.message) { - // eslint-disable-next-line no-console -- Debugging output - console.log(error.message); - } else { - // eslint-disable-next-line no-console -- Debugging output - console.log(error); - } + reportError(error); } } @@ -84,13 +94,27 @@ class CoderReporter implements Reporter { } } } - testOutput.delete(test.id); + this.testOutput.delete(test.id); + await exportDebugPprof(test.title); } onEnd(result: FullResult) { // eslint-disable-next-line no-console -- Helpful for debugging console.log(`==> Tests ${result.status}`); + console.log(`${this.passedCount} passed`); + if (this.failedTests.length > 0) { + console.log(`${this.failedTests.length} failed`); + for (const test of this.failedTests) { + console.log(` ${test.location.file} › ${test.title}`); + } + } + if (this.timedOutTests.length > 0) { + console.log(`${this.timedOutTests.length} timed out`); + for (const test of this.timedOutTests) { + console.log(` ${test.location.file} › ${test.title}`); + } + } } } @@ -119,5 +143,24 @@ const exportDebugPprof = async (testName: string) => { }); }; +const reportError = (error: TestError) => { + if (error.location) { + // eslint-disable-next-line no-console -- Debugging output + console.log(`${error.location.file}:${error.location.line}:`); + } + if (error.snippet) { + // eslint-disable-next-line no-console -- Debugging output + console.log(error.snippet); + } + + if (error.message) { + // eslint-disable-next-line no-console -- Debugging output + console.log(error.message); + } else { + // eslint-disable-next-line no-console -- Debugging output + console.log(error); + } +}; + // eslint-disable-next-line no-unused-vars -- Playwright config uses it export default CoderReporter; From 335593a6d53eab8ddadeeb2720b07673c7728aee Mon Sep 17 00:00:00 2001 From: McKayla Washburn Date: Mon, 16 Oct 2023 19:39:43 +0000 Subject: [PATCH 4/7] =?UTF-8?q?=F0=9F=A7=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- site/.eslintrc.yaml | 3 +++ site/e2e/reporter.ts | 17 ++++------------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/site/.eslintrc.yaml b/site/.eslintrc.yaml index ac7c6936ff5a6..478c1e070415e 100644 --- a/site/.eslintrc.yaml +++ b/site/.eslintrc.yaml @@ -91,6 +91,9 @@ rules: - allowSingleExtends: true "brace-style": "off" "curly": ["error", "all"] + "eslint-comments/disable-enable-pair": + - error + - allowWholeFile: true "eslint-comments/require-description": "error" eqeqeq: error import/default: "off" diff --git a/site/e2e/reporter.ts b/site/e2e/reporter.ts index bc6fd32d96cdc..b08c63aedb8a4 100644 --- a/site/e2e/reporter.ts +++ b/site/e2e/reporter.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-console -- Logging is sort of the whole point here */ import * as fs from "fs"; import type { FullConfig, @@ -20,13 +21,11 @@ class CoderReporter implements Reporter { onBegin(config: FullConfig, suite: Suite) { this.config = config; - // eslint-disable-next-line no-console -- Helpful for debugging console.log(`==> Running ${suite.allTests().length} tests`); } onTestBegin(test: TestCase) { this.testOutput.set(test.id, []); - // eslint-disable-next-line no-console -- Helpful for debugging console.log(`==> Starting test ${test.title}`); } @@ -53,7 +52,7 @@ class CoderReporter implements Reporter { } async onTestEnd(test: TestCase, result: TestResult) { - console.log(`==> Finished test ${test.title}: ${result.status}`); // eslint-disable-line no-console -- Helpful for debugging + console.log(`==> Finished test ${test.title}: ${result.status}`); if (result.status === "passed") { this.passedCount++; @@ -72,24 +71,22 @@ class CoderReporter implements Reporter { preserve === "always" || (result.status !== "passed" && preserve !== "never"); if (logOutput) { - console.log("==> Output"); // eslint-disable-line no-console -- Debugging output + console.log("==> Output"); const output = this.testOutput.get(test.id)!; for (const [target, chunk] of output) { target.write(`${chunk.replace(/\n$/g, "")}\n`); } if (result.errors.length > 0) { - console.log("==> Errors"); // eslint-disable-line no-console -- Debugging output + console.log("==> Errors"); for (const error of result.errors) { reportError(error); } } if (result.attachments.length > 0) { - // eslint-disable-next-line no-console -- Debugging output console.log("==> Attachments"); for (const attachment of result.attachments) { - // eslint-disable-next-line no-console -- Debugging output console.log(attachment); } } @@ -100,7 +97,6 @@ class CoderReporter implements Reporter { } onEnd(result: FullResult) { - // eslint-disable-next-line no-console -- Helpful for debugging console.log(`==> Tests ${result.status}`); console.log(`${this.passedCount} passed`); if (this.failedTests.length > 0) { @@ -133,7 +129,6 @@ const exportDebugPprof = async (testName: string) => { if (err) { throw new Error(`Error writing to ${outputFile}: ${err.message}`); } else { - // eslint-disable-next-line no-console -- Helpful for debugging console.log(`Data from ${url} has been saved to ${outputFile}`); } }); @@ -145,19 +140,15 @@ const exportDebugPprof = async (testName: string) => { const reportError = (error: TestError) => { if (error.location) { - // eslint-disable-next-line no-console -- Debugging output console.log(`${error.location.file}:${error.location.line}:`); } if (error.snippet) { - // eslint-disable-next-line no-console -- Debugging output console.log(error.snippet); } if (error.message) { - // eslint-disable-next-line no-console -- Debugging output console.log(error.message); } else { - // eslint-disable-next-line no-console -- Debugging output console.log(error); } }; From 91475e468fed24c49efed415fb4c4b59357f69ec Mon Sep 17 00:00:00 2001 From: McKayla Washburn Date: Mon, 16 Oct 2023 16:09:30 -0600 Subject: [PATCH 5/7] =?UTF-8?q?=F0=9F=A7=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- site/e2e/reporter.ts | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/site/e2e/reporter.ts b/site/e2e/reporter.ts index b08c63aedb8a4..a97ad95979f03 100644 --- a/site/e2e/reporter.ts +++ b/site/e2e/reporter.ts @@ -1,5 +1,5 @@ /* eslint-disable no-console -- Logging is sort of the whole point here */ -import * as fs from "fs"; +import * as fs from "fs/promises"; import type { FullConfig, Suite, @@ -66,11 +66,15 @@ class CoderReporter implements Reporter { this.timedOutTests.push(test); } + const outputFile = `test-results/debug-pprof-goroutine-${test.title}.txt`; + await exportDebugPprof(outputFile); + const preserve = this.config?.preserveOutput; const logOutput = preserve === "always" || (result.status !== "passed" && preserve !== "never"); if (logOutput) { + console.log(`Data from pprof has been saved to ${outputFile}`); console.log("==> Output"); const output = this.testOutput.get(test.id)!; for (const [target, chunk] of output) { @@ -92,8 +96,6 @@ class CoderReporter implements Reporter { } } this.testOutput.delete(test.id); - - await exportDebugPprof(test.title); } onEnd(result: FullResult) { @@ -114,28 +116,15 @@ class CoderReporter implements Reporter { } } -const exportDebugPprof = async (testName: string) => { - const url = "http://127.0.0.1:6060/debug/pprof/goroutine?debug=1"; - const outputFile = `test-results/debug-pprof-goroutine-${testName}.txt`; - - await axios - .get(url) - .then((response) => { - if (response.status !== 200) { - throw new Error(`Error: Received status code ${response.status}`); - } +const exportDebugPprof = async (outputFile: string) => { + const response = await axios.get( + "http://127.0.0.1:6060/debug/pprof/goroutine?debug=1", + ); + if (response.status !== 200) { + throw new Error(`Error: Received status code ${response.status}`); + } - fs.writeFile(outputFile, response.data, (err) => { - if (err) { - throw new Error(`Error writing to ${outputFile}: ${err.message}`); - } else { - console.log(`Data from ${url} has been saved to ${outputFile}`); - } - }); - }) - .catch((error) => { - throw new Error(`Error: ${error.message}`); - }); + await fs.writeFile(outputFile, response.data); }; const reportError = (error: TestError) => { From 395a019ea13c40a1ee99555c12d0db9033763a07 Mon Sep 17 00:00:00 2001 From: McKayla Washburn Date: Tue, 17 Oct 2023 16:09:50 +0000 Subject: [PATCH 6/7] let's see what this looks like --- site/e2e/playwright.config.ts | 3 +-- site/e2e/reporter.ts | 19 +++++-------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/site/e2e/playwright.config.ts b/site/e2e/playwright.config.ts index 7782ad5245e87..dcd82a6dfd999 100644 --- a/site/e2e/playwright.config.ts +++ b/site/e2e/playwright.config.ts @@ -6,7 +6,7 @@ export const port = process.env.CODER_E2E_PORT ? Number(process.env.CODER_E2E_PORT) : defaultPort; -const coderMain = path.join(__dirname, "../../enterprise/cmd/coder/main.go"); +const coderMain = path.join(__dirname, "../../enterprise/cmd/coder"); export const STORAGE_STATE = path.join(__dirname, ".auth.json"); @@ -15,7 +15,6 @@ const localURL = (port: number, path: string): string => { }; export default defineConfig({ - preserveOutput: "failures-only", projects: [ { name: "setup", diff --git a/site/e2e/reporter.ts b/site/e2e/reporter.ts index a97ad95979f03..26ecff82871a0 100644 --- a/site/e2e/reporter.ts +++ b/site/e2e/reporter.ts @@ -31,10 +31,7 @@ class CoderReporter implements Reporter { onStdOut(chunk: string, test?: TestCase, _?: TestResult): void { if (!test) { - const preserve = this.config?.preserveOutput === "always"; - if (preserve) { - console.log(`[stdout] ${chunk.replace(/\n$/g, "")}`); - } + console.log(`[stdout] ${chunk.replace(/\n$/g, "")}`); return; } this.testOutput.get(test.id)!.push([process.stdout, chunk]); @@ -42,10 +39,7 @@ class CoderReporter implements Reporter { onStdErr(chunk: string, test?: TestCase, _?: TestResult): void { if (!test) { - const preserve = this.config?.preserveOutput === "always"; - if (preserve) { - console.error(`[stderr] ${chunk.replace(/\n$/g, "")}`); - } + console.error(`[stderr] ${chunk.replace(/\n$/g, "")}`); return; } this.testOutput.get(test.id)!.push([process.stderr, chunk]); @@ -66,14 +60,11 @@ class CoderReporter implements Reporter { this.timedOutTests.push(test); } - const outputFile = `test-results/debug-pprof-goroutine-${test.title}.txt`; + const fsTestTitle = test.title.replaceAll(" ", "-"); + const outputFile = `test-results/debug-pprof-goroutine-${fsTestTitle}.txt`; await exportDebugPprof(outputFile); - const preserve = this.config?.preserveOutput; - const logOutput = - preserve === "always" || - (result.status !== "passed" && preserve !== "never"); - if (logOutput) { + if (result.status !== "passed") { console.log(`Data from pprof has been saved to ${outputFile}`); console.log("==> Output"); const output = this.testOutput.get(test.id)!; From 1c6ff274b17285d2f9f9d77ab2a0cec6052b2870 Mon Sep 17 00:00:00 2001 From: McKayla Washburn Date: Tue, 17 Oct 2023 21:03:02 +0000 Subject: [PATCH 7/7] ignore some lines --- site/e2e/reporter.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/site/e2e/reporter.ts b/site/e2e/reporter.ts index 26ecff82871a0..6be742b02bb55 100644 --- a/site/e2e/reporter.ts +++ b/site/e2e/reporter.ts @@ -31,7 +31,9 @@ class CoderReporter implements Reporter { onStdOut(chunk: string, test?: TestCase, _?: TestResult): void { if (!test) { - console.log(`[stdout] ${chunk.replace(/\n$/g, "")}`); + for (const line of filteredServerLogLines(chunk)) { + console.log(`[stdout] ${line}`); + } return; } this.testOutput.get(test.id)!.push([process.stdout, chunk]); @@ -39,7 +41,9 @@ class CoderReporter implements Reporter { onStdErr(chunk: string, test?: TestCase, _?: TestResult): void { if (!test) { - console.error(`[stderr] ${chunk.replace(/\n$/g, "")}`); + for (const line of filteredServerLogLines(chunk)) { + console.error(`[stderr] ${line}`); + } return; } this.testOutput.get(test.id)!.push([process.stderr, chunk]); @@ -107,6 +111,12 @@ class CoderReporter implements Reporter { } } +const shouldPrintLine = (line: string) => + [" error=EOF", "coderd: audit_log"].every((noise) => !line.includes(noise)); + +const filteredServerLogLines = (chunk: string): string[] => + chunk.trimEnd().split("\n").filter(shouldPrintLine); + const exportDebugPprof = async (outputFile: string) => { const response = await axios.get( "http://127.0.0.1:6060/debug/pprof/goroutine?debug=1",