fix(testing): reduce flakiness in extension e2e tests (#5481)

jsjoeio · web-flow · commit 8352a22e33c2 · 2022-08-22T12:47:36.000-07:00
diff --git a/test/e2e/extensions.test.ts b/test/e2e/extensions.test.ts
@@ -8,12 +8,14 @@ function runTestExtensionTests() {
   test("should have access to VSCODE_PROXY_URI", async ({ codeServerPage }) => {
     const address = await getMaybeProxiedCodeServer(codeServerPage)
 
+    await codeServerPage.waitForTestExtensionLoaded()
     await codeServerPage.executeCommandViaMenus("code-server: Get proxy URI")
 
-    const text = await codeServerPage.page.locator(".notification-list-item-message").textContent()
+    await codeServerPage.page.waitForSelector("text=proxyUri", { timeout: 3000 })
+    const text = await codeServerPage.page.locator("text=proxyUri").first().textContent()
     // Remove end slash in address
     const normalizedAddress = address.replace(/\/+$/, "")
-    expect(text).toBe(`${normalizedAddress}/proxy/{{port}}`)
+    expect(text).toBe(`Info: proxyUri: ${normalizedAddress}/proxy/{{port}}`)
   })
 }
 
diff --git a/test/e2e/extensions/test-extension/extension.ts b/test/e2e/extensions/test-extension/extension.ts
@@ -1,10 +1,11 @@
 import * as vscode from "vscode"
 
 export function activate(context: vscode.ExtensionContext) {
+  vscode.window.showInformationMessage("test extension loaded")
   context.subscriptions.push(
     vscode.commands.registerCommand("codeServerTest.proxyUri", () => {
       if (process.env.VSCODE_PROXY_URI) {
-        vscode.window.showInformationMessage(process.env.VSCODE_PROXY_URI)
+        vscode.window.showInformationMessage(`proxyUri: ${process.env.VSCODE_PROXY_URI}`)
       } else {
         vscode.window.showErrorMessage("No proxy URI was set")
       }
diff --git a/test/e2e/extensions/test-extension/package.json b/test/e2e/extensions/test-extension/package.json
@@ -4,7 +4,7 @@
   "version": "0.0.1",
   "publisher": "coder",
   "activationEvents": [
-    "onCommand:codeServerTest.proxyUri"
+    "onStartupFinished"
   ],
   "engines": {
     "vscode": "^1.56.0"
diff --git a/test/e2e/models/CodeServer.ts b/test/e2e/models/CodeServer.ts
@@ -296,6 +296,16 @@ export class CodeServerPage {
     return visible
   }
 
+  /**
+   * Checks if the test extension loaded
+   */
+  async waitForTestExtensionLoaded(): Promise<void> {
+    const selector = "text=test extension loaded"
+    this.codeServer.logger.debug("Waiting for test extension to load...")
+
+    await this.page.waitForSelector(selector)
+  }
+
   /**
    * Focuses the integrated terminal by navigating through the command palette.
    *
