report specific test failure names

aslilac · aslilac · commit 011dc55c710d · 2023-10-16T19:24:35.000Z
diff --git a/site/e2e/playwright.config.ts b/site/e2e/playwright.config.ts
@@ -15,6 +15,7 @@ const localURL = (port: number, path: string): string => {
 };
 
 export default defineConfig({
+  preserveOutput: "failures-only",
   projects: [
     {
       name: "setup",
diff --git a/site/e2e/reporter.ts b/site/e2e/reporter.ts
@@ -6,72 +6,82 @@ import type {
   TestResult,
   FullResult,
   Reporter,
+  TestError,
 } from "@playwright/test/reporter";
 import axios from "axios";
 import type { Writable } from "stream";
 
-const testOutput = new Map<string, Array<[Writable, string]>>();
-
 class CoderReporter implements Reporter {
+  config: FullConfig | null = null;
+  testOutput = new Map<string, Array<[Writable, string]>>();
+  passedCount = 0;
+  failedTests: TestCase[] = [];
+  timedOutTests: TestCase[] = [];
+
   onBegin(config: FullConfig, suite: Suite) {
+    this.config = config;
     // eslint-disable-next-line no-console -- Helpful for debugging
     console.log(`==> Running ${suite.allTests().length} tests`);
   }
 
   onTestBegin(test: TestCase) {
-    testOutput.set(test.id, []);
+    this.testOutput.set(test.id, []);
     // eslint-disable-next-line no-console -- Helpful for debugging
     console.log(`==> Starting test ${test.title}`);
   }
 
   onStdOut(chunk: string, test?: TestCase, _?: TestResult): void {
     if (!test) {
-      // console.log(`[stdout] [unknown] ${chunk.replace(/\n$/g, "")}`);
+      const preserve = this.config?.preserveOutput === "always";
+      if (preserve) {
+        console.log(`[stdout] ${chunk.replace(/\n$/g, "")}`);
+      }
       return;
     }
-    testOutput.get(test.id)!.push([process.stdout, chunk]);
+    this.testOutput.get(test.id)!.push([process.stdout, chunk]);
   }
 
   onStdErr(chunk: string, test?: TestCase, _?: TestResult): void {
     if (!test) {
-      // console.error(`[stderr] [unknown] ${chunk.replace(/\n$/g, "")}`);
+      const preserve = this.config?.preserveOutput === "always";
+      if (preserve) {
+        console.error(`[stderr] ${chunk.replace(/\n$/g, "")}`);
+      }
       return;
     }
-    testOutput.get(test.id)!.push([process.stderr, chunk]);
+    this.testOutput.get(test.id)!.push([process.stderr, chunk]);
   }
 
   async onTestEnd(test: TestCase, result: TestResult) {
-    // eslint-disable-next-line no-console -- Helpful for debugging
-    console.log(`==> Finished test ${test.title}: ${result.status}`);
+    console.log(`==> Finished test ${test.title}: ${result.status}`); // eslint-disable-line no-console -- Helpful for debugging
+
+    if (result.status === "passed") {
+      this.passedCount++;
+    }
+
+    if (result.status === "failed") {
+      this.failedTests.push(test);
+    }
 
-    if (result.status !== "passed") {
-      // eslint-disable-next-line no-console -- Debugging output
-      console.log("==> Output");
-      const output = testOutput.get(test.id)!;
+    if (result.status === "timedOut") {
+      this.timedOutTests.push(test);
+    }
+
+    const preserve = this.config?.preserveOutput;
+    const logOutput =
+      preserve === "always" ||
+      (result.status !== "passed" && preserve !== "never");
+    if (logOutput) {
+      console.log("==> Output"); // eslint-disable-line no-console -- Debugging output
+      const output = this.testOutput.get(test.id)!;
       for (const [target, chunk] of output) {
         target.write(`${chunk.replace(/\n$/g, "")}\n`);
       }
 
       if (result.errors.length > 0) {
-        // eslint-disable-next-line no-console -- Debugging output
-        console.log("==> Errors");
+        console.log("==> Errors"); // eslint-disable-line no-console -- Debugging output
         for (const error of result.errors) {
-          if (error.location) {
-            // eslint-disable-next-line no-console -- Debugging output
-            console.log(`${error.location.file}:${error.location.line}:`);
-          }
-          if (error.snippet) {
-            // eslint-disable-next-line no-console -- Debugging output
-            console.log(error.snippet);
-          }
-
-          if (error.message) {
-            // eslint-disable-next-line no-console -- Debugging output
-            console.log(error.message);
-          } else {
-            // eslint-disable-next-line no-console -- Debugging output
-            console.log(error);
-          }
+          reportError(error);
         }
       }
 
@@ -84,13 +94,27 @@ class CoderReporter implements Reporter {
         }
       }
     }
-    testOutput.delete(test.id);
+    this.testOutput.delete(test.id);
+
     await exportDebugPprof(test.title);
   }
 
   onEnd(result: FullResult) {
     // eslint-disable-next-line no-console -- Helpful for debugging
     console.log(`==> Tests ${result.status}`);
+    console.log(`${this.passedCount} passed`);
+    if (this.failedTests.length > 0) {
+      console.log(`${this.failedTests.length} failed`);
+      for (const test of this.failedTests) {
+        console.log(`  ${test.location.file} › ${test.title}`);
+      }
+    }
+    if (this.timedOutTests.length > 0) {
+      console.log(`${this.timedOutTests.length} timed out`);
+      for (const test of this.timedOutTests) {
+        console.log(`  ${test.location.file} › ${test.title}`);
+      }
+    }
   }
 }
 
@@ -119,5 +143,24 @@ const exportDebugPprof = async (testName: string) => {
     });
 };
 
+const reportError = (error: TestError) => {
+  if (error.location) {
+    // eslint-disable-next-line no-console -- Debugging output
+    console.log(`${error.location.file}:${error.location.line}:`);
+  }
+  if (error.snippet) {
+    // eslint-disable-next-line no-console -- Debugging output
+    console.log(error.snippet);
+  }
+
+  if (error.message) {
+    // eslint-disable-next-line no-console -- Debugging output
+    console.log(error.message);
+  } else {
+    // eslint-disable-next-line no-console -- Debugging output
+    console.log(error);
+  }
+};
+
 // eslint-disable-next-line no-unused-vars -- Playwright config uses it
 export default CoderReporter;

Original file line number	Diff line number	Diff line change
`@@ -15,6 +15,7 @@ const localURL = (port: number, path: string): string => {`
`15`	`15`	`};`
`16`	`16`
`17`	`17`	`export default defineConfig({`
	`18`	`+ preserveOutput: "failures-only",`
`18`	`19`	`projects: [`
`19`	`20`	`{`
`20`	`21`	`name: "setup",`