coder · aslilac · Oct 17, 2023 · Oct 16, 2023 · Oct 16, 2023 · Oct 16, 2023
diff --git a/site/.eslintrc.yaml b/site/.eslintrc.yaml
@@ -91,6 +91,9 @@ rules:
     - allowSingleExtends: true
   "brace-style": "off"
   "curly": ["error", "all"]
+  "eslint-comments/disable-enable-pair":
+    - error
+    - allowWholeFile: true
   "eslint-comments/require-description": "error"
   eqeqeq: error
   import/default: "off"

diff --git a/site/e2e/playwright.config.ts b/site/e2e/playwright.config.ts
@@ -6,7 +6,7 @@ export const port = process.env.CODER_E2E_PORT
   ? Number(process.env.CODER_E2E_PORT)
   : defaultPort;
 
-const coderMain = path.join(__dirname, "../../enterprise/cmd/coder/main.go");
+const coderMain = path.join(__dirname, "../../enterprise/cmd/coder");
 
 export const STORAGE_STATE = path.join(__dirname, ".auth.json");
 

diff --git a/site/e2e/reporter.ts b/site/e2e/reporter.ts
@@ -1,85 +1,146 @@
-import fs from "fs";
+/* eslint-disable no-console -- Logging is sort of the whole point here */
+import * as fs from "fs/promises";
 import type {
   FullConfig,
   Suite,
   TestCase,
   TestResult,
   FullResult,
   Reporter,
+  TestError,
 } from "@playwright/test/reporter";
 import axios from "axios";
+import type { Writable } from "stream";
 
 class CoderReporter implements Reporter {
+  config: FullConfig | null = null;
+  testOutput = new Map<string, Array<[Writable, string]>>();
+  passedCount = 0;
+  failedTests: TestCase[] = [];
+  timedOutTests: TestCase[] = [];
+
   onBegin(config: FullConfig, suite: Suite) {
-    // eslint-disable-next-line no-console -- Helpful for debugging
-    console.log(`Starting the run with ${suite.allTests().length} tests`);
+    this.config = config;
+    console.log(`==> Running ${suite.allTests().length} tests`);
   }
 
   onTestBegin(test: TestCase) {
-    // eslint-disable-next-line no-console -- Helpful for debugging
-    console.log(`Starting test ${test.title}`);
+    this.testOutput.set(test.id, []);
+    console.log(`==> Starting test ${test.title}`);
   }
 
-  onStdOut(chunk: string, test: TestCase, _: TestResult): void {
-    // eslint-disable-next-line no-console -- Helpful for debugging
-    console.log(
-      `[stdout] [${test ? test.title : "unknown"}]: ${chunk.replace(
-        /\n$/g,
-        "",
-      )}`,
-    );
+  onStdOut(chunk: string, test?: TestCase, _?: TestResult): void {
+    if (!test) {
+      for (const line of filteredServerLogLines(chunk)) {
+        console.log(`[stdout] ${line}`);
+      }
+      return;
+    }
+    this.testOutput.get(test.id)!.push([process.stdout, chunk]);
   }
 
-  onStdErr(chunk: string, test: TestCase, _: TestResult): void {
-    // eslint-disable-next-line no-console -- Helpful for debugging
-    console.log(
-      `[stderr] [${test ? test.title : "unknown"}]: ${chunk.replace(
-        /\n$/g,
-        "",
-      )}`,
-    );
+  onStdErr(chunk: string, test?: TestCase, _?: TestResult): void {
+    if (!test) {
+      for (const line of filteredServerLogLines(chunk)) {
+        console.error(`[stderr] ${line}`);
+      }
+      return;
+    }
+    this.testOutput.get(test.id)!.push([process.stderr, chunk]);
   }
 
   async onTestEnd(test: TestCase, result: TestResult) {
-    // eslint-disable-next-line no-console -- Helpful for debugging
-    console.log(`Finished test ${test.title}: ${result.status}`);
+    console.log(`==> Finished test ${test.title}: ${result.status}`);
+
+    if (result.status === "passed") {
+      this.passedCount++;
+    }
+
+    if (result.status === "failed") {
+      this.failedTests.push(test);
+    }
+
+    if (result.status === "timedOut") {
+      this.timedOutTests.push(test);
+    }
+
+    const fsTestTitle = test.title.replaceAll(" ", "-");
+    const outputFile = `test-results/debug-pprof-goroutine-${fsTestTitle}.txt`;
+    await exportDebugPprof(outputFile);
 
     if (result.status !== "passed") {
-      // eslint-disable-next-line no-console -- Helpful for debugging
-      console.log("errors", result.errors, "attachments", result.attachments);
+      console.log(`Data from pprof has been saved to ${outputFile}`);
+      console.log("==> Output");
+      const output = this.testOutput.get(test.id)!;
+      for (const [target, chunk] of output) {
+        target.write(`${chunk.replace(/\n$/g, "")}\n`);
+      }
+
+      if (result.errors.length > 0) {
+        console.log("==> Errors");
+        for (const error of result.errors) {
+          reportError(error);
+        }
+      }
+
+      if (result.attachments.length > 0) {
+        console.log("==> Attachments");
+        for (const attachment of result.attachments) {
+          console.log(attachment);
+        }
+      }
     }
-    await exportDebugPprof(test.title);
+    this.testOutput.delete(test.id);
   }
 
   onEnd(result: FullResult) {
-    // eslint-disable-next-line no-console -- Helpful for debugging
-    console.log(`Finished the run: ${result.status}`);
+    console.log(`==> Tests ${result.status}`);
+    console.log(`${this.passedCount} passed`);
+    if (this.failedTests.length > 0) {
+      console.log(`${this.failedTests.length} failed`);
+      for (const test of this.failedTests) {
+        console.log(`  ${test.location.file} › ${test.title}`);
+      }
+    }
+    if (this.timedOutTests.length > 0) {
+      console.log(`${this.timedOutTests.length} timed out`);
+      for (const test of this.timedOutTests) {
+        console.log(`  ${test.location.file} › ${test.title}`);
+      }
+    }
   }
 }
 
-const exportDebugPprof = async (testName: string) => {
-  const url = "http://127.0.0.1:6060/debug/pprof/goroutine?debug=1";
-  const outputFile = `test-results/debug-pprof-goroutine-${testName}.txt`;
+const shouldPrintLine = (line: string) =>
+  ["  error=EOF", "coderd: audit_log"].every((noise) => !line.includes(noise));
 
-  await axios
-    .get(url)
-    .then((response) => {
-      if (response.status !== 200) {
-        throw new Error(`Error: Received status code ${response.status}`);
-      }
+const filteredServerLogLines = (chunk: string): string[] =>
+  chunk.trimEnd().split("\n").filter(shouldPrintLine);
 
-      fs.writeFile(outputFile, response.data, (err) => {
-        if (err) {
-          throw new Error(`Error writing to ${outputFile}: ${err.message}`);
-        } else {
-          // eslint-disable-next-line no-console -- Helpful for debugging
-          console.log(`Data from ${url} has been saved to ${outputFile}`);
-        }
-      });
-    })
-    .catch((error) => {
-      throw new Error(`Error: ${error.message}`);
-    });
+const exportDebugPprof = async (outputFile: string) => {
+  const response = await axios.get(
+    "http://127.0.0.1:6060/debug/pprof/goroutine?debug=1",
+  );
+  if (response.status !== 200) {
+    throw new Error(`Error: Received status code ${response.status}`);
+  }
+
+  await fs.writeFile(outputFile, response.data);
+};
+
+const reportError = (error: TestError) => {
+  if (error.location) {
+    console.log(`${error.location.file}:${error.location.line}:`);
+  }
+  if (error.snippet) {
+    console.log(error.snippet);
+  }
+
+  if (error.message) {
+    console.log(error.message);
+  } else {
+    console.log(error);
+  }
 };
 
 // eslint-disable-next-line no-unused-vars -- Playwright config uses it