Skip to content

Commit 56f5f4b

Browse files
mtojekpull[bot]
mtojek
authored and
pull[bot]
committed
test: improve E2E framework (#9469)
1 parent 5c7fd68 commit 56f5f4b

File tree

12 files changed

+133
-27
lines changed

12 files changed

+133
-27
lines changed

‎.github/workflows/ci.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -567,6 +567,14 @@ jobs:
567567
path: ./site/test-results/**/*.webm
568568
retention-days: 7
569569

570+
- name: Upload pprof dumps
571+
if: always() && github.actor != 'dependabot[bot]' && runner.os == 'Linux' && !github.event.pull_request.head.repo.fork
572+
uses: actions/upload-artifact@v3
573+
with:
574+
name: debug-pprof-dumps
575+
path: ./site/test-results/**/debug-pprof-*.txt
576+
retention-days: 7
577+
570578
chromatic:
571579
# REMARK: this is only used to build storybook and deploy it to Chromatic.
572580
runs-on: ubuntu-latest

‎site/e2e/constants.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
// Default port from the server
22
export const defaultPort = 3000
3+
export const prometheusPort = 2114
4+
export const pprofPort = 6061
35

46
// Credentials for the first user
57
export const username = "admin"

‎site/e2e/global.setup.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { STORAGE_STATE } from "./playwright.config"
44
import { Language } from "../src/components/CreateUserForm/CreateUserForm"
55

66
test("create first user", async ({ page }) => {
7-
await page.goto("/", { waitUntil: "networkidle" })
7+
await page.goto("/", { waitUntil: "domcontentloaded" })
88

99
await page.getByLabel(Language.usernameLabel).fill(constants.username)
1010
await page.getByLabel(Language.emailLabel).fill(constants.email)

‎site/e2e/helpers.ts

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { expect, Page } from "@playwright/test"
2-
import { spawn } from "child_process"
2+
import { ChildProcess, exec, spawn } from "child_process"
33
import { randomUUID } from "crypto"
44
import path from "path"
55
import express from "express"
@@ -15,10 +15,12 @@ import {
1515
Resource,
1616
RichParameter,
1717
} from "./provisionerGenerated"
18+
import { prometheusPort, pprofPort } from "./constants"
1819
import { port } from "./playwright.config"
1920
import * as ssh from "ssh2"
2021
import { Duplex } from "stream"
2122
import { WorkspaceBuildParameter } from "api/typesGenerated"
23+
import axios from "axios"
2224

2325
// createWorkspace creates a workspace for a template.
2426
// It does not wait for it to be running, but it does navigate to the page.
@@ -29,7 +31,7 @@ export const createWorkspace = async (
2931
buildParameters: WorkspaceBuildParameter[] = [],
3032
): Promise<string> => {
3133
await page.goto("/templates/" + templateName + "/workspace", {
32-
waitUntil: "networkidle",
34+
waitUntil: "domcontentloaded",
3335
})
3436
await expect(page).toHaveURL("/templates/" + templateName + "/workspace")
3537

@@ -57,7 +59,7 @@ export const verifyParameters = async (
5759
expectedBuildParameters: WorkspaceBuildParameter[],
5860
) => {
5961
await page.goto("/@admin/" + workspaceName + "/settings/parameters", {
60-
waitUntil: "networkidle",
62+
waitUntil: "domcontentloaded",
6163
})
6264
await expect(page).toHaveURL(
6365
"/@admin/" + workspaceName + "/settings/parameters",
@@ -120,7 +122,7 @@ export const createTemplate = async (
120122
content: "window.playwright = true",
121123
})
122124

123-
await page.goto("/templates/new", { waitUntil: "networkidle" })
125+
await page.goto("/templates/new", { waitUntil: "domcontentloaded" })
124126
await expect(page).toHaveURL("/templates/new")
125127

126128
await page.getByTestId("file-upload").setInputFiles({
@@ -229,7 +231,10 @@ export const buildWorkspaceWithParameters = async (
229231

230232
// startAgent runs the coder agent with the provided token.
231233
// It awaits the agent to be ready before returning.
232-
export const startAgent = async (page: Page, token: string): Promise<void> => {
234+
export const startAgent = async (
235+
page: Page,
236+
token: string,
237+
): Promise<ChildProcess> => {
233238
return startAgentWithCommand(page, token, "go", "run", coderMainPath())
234239
}
235240

@@ -308,14 +313,14 @@ export const startAgentWithCommand = async (
308313
token: string,
309314
command: string,
310315
...args: string[]
311-
): Promise<void> => {
316+
): Promise<ChildProcess> => {
312317
const cp = spawn(command, [...args, "agent", "--no-reap"], {
313318
env: {
314319
...process.env,
315320
CODER_AGENT_URL: "http://localhost:" + port,
316321
CODER_AGENT_TOKEN: token,
317-
CODER_AGENT_PPROF_ADDRESS: "127.0.0.1:2114",
318-
CODER_AGENT_PROMETHEUS_ADDRESS: "127.0.0.1:6061",
322+
CODER_AGENT_PPROF_ADDRESS: "127.0.0.1:" + pprofPort,
323+
CODER_AGENT_PROMETHEUS_ADDRESS: "127.0.0.1:" + prometheusPort,
319324
},
320325
})
321326
cp.stdout.on("data", (data: Buffer) => {
@@ -332,6 +337,39 @@ export const startAgentWithCommand = async (
332337
})
333338

334339
await page.getByTestId("agent-status-ready").waitFor({ state: "visible" })
340+
return cp
341+
}
342+
343+
export const stopAgent = async (cp: ChildProcess, goRun: boolean = true) => {
344+
// When the web server is started with `go run`, it spawns a child process with coder server.
345+
// `pkill -P` terminates child processes belonging the same group as `go run`.
346+
// The command `kill` is used to terminate a web server started as a standalone binary.
347+
exec(goRun ? `pkill -P ${cp.pid}` : `kill ${cp.pid}`, (error) => {
348+
if (error) {
349+
throw new Error(`exec error: ${JSON.stringify(error)}`)
350+
}
351+
})
352+
await waitUntilUrlIsNotResponding("http://localhost:" + prometheusPort)
353+
}
354+
355+
const waitUntilUrlIsNotResponding = async (url: string) => {
356+
const maxRetries = 30
357+
const retryIntervalMs = 1000
358+
let retries = 0
359+
360+
while (retries < maxRetries) {
361+
try {
362+
await axios.get(url)
363+
} catch (error) {
364+
return
365+
}
366+
367+
retries++
368+
await new Promise((resolve) => setTimeout(resolve, retryIntervalMs))
369+
}
370+
throw new Error(
371+
`URL ${url} is still responding after ${maxRetries * retryIntervalMs}ms`,
372+
)
335373
}
336374

337375
const coderMainPath = (): string => {

‎site/e2e/playwright.config.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ export default defineConfig({
4646
`--dangerous-disable-rate-limits ` +
4747
`--provisioner-daemons 10 ` +
4848
`--provisioner-daemons-echo ` +
49-
`--provisioner-daemon-poll-interval 50ms`,
49+
`--provisioner-daemon-poll-interval 50ms ` +
50+
`--pprof-enable`,
5051
env: {
5152
...process.env,
5253

‎site/e2e/reporter.ts

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import fs from "fs"
12
import type {
23
FullConfig,
34
Suite,
@@ -6,6 +7,7 @@ import type {
67
FullResult,
78
Reporter,
89
} from "@playwright/test/reporter"
10+
import axios from "axios"
911

1012
class CoderReporter implements Reporter {
1113
onBegin(config: FullConfig, suite: Suite) {
@@ -38,13 +40,15 @@ class CoderReporter implements Reporter {
3840
)
3941
}
4042

41-
onTestEnd(test: TestCase, result: TestResult) {
43+
async onTestEnd(test: TestCase, result: TestResult) {
4244
// eslint-disable-next-line no-console -- Helpful for debugging
4345
console.log(`Finished test ${test.title}: ${result.status}`)
46+
4447
if (result.status !== "passed") {
4548
// eslint-disable-next-line no-console -- Helpful for debugging
4649
console.log("errors", result.errors, "attachments", result.attachments)
4750
}
51+
await exportDebugPprof(test.title)
4852
}
4953

5054
onEnd(result: FullResult) {
@@ -53,5 +57,30 @@ class CoderReporter implements Reporter {
5357
}
5458
}
5559

60+
const exportDebugPprof = async (testName: string) => {
61+
const url = "http://127.0.0.1:6060/debug/pprof/goroutine?debug=1"
62+
const outputFile = `test-results/debug-pprof-goroutine-${testName}.txt`
63+
64+
await axios
65+
.get(url)
66+
.then((response) => {
67+
if (response.status !== 200) {
68+
throw new Error(`Error: Received status code ${response.status}`)
69+
}
70+
71+
fs.writeFile(outputFile, response.data, (err) => {
72+
if (err) {
73+
throw new Error(`Error writing to ${outputFile}: ${err.message}`)
74+
} else {
75+
// eslint-disable-next-line no-console -- Helpful for debugging
76+
console.log(`Data from ${url} has been saved to ${outputFile}`)
77+
}
78+
})
79+
})
80+
.catch((error) => {
81+
throw new Error(`Error: ${error.message}`)
82+
})
83+
}
84+
5685
// eslint-disable-next-line no-unused-vars -- Playwright config uses it
5786
export default CoderReporter

‎site/e2e/tests/app.spec.ts

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,13 @@
11
import { test } from "@playwright/test"
22
import { randomUUID } from "crypto"
33
import * as http from "http"
4-
import { createTemplate, createWorkspace, startAgent } from "../helpers"
4+
import {
5+
createTemplate,
6+
createWorkspace,
7+
startAgent,
8+
stopAgent,
9+
stopWorkspace,
10+
} from "../helpers"
511
import { beforeCoderTest } from "../hooks"
612

713
test.beforeEach(async ({ page }) => await beforeCoderTest(page))
@@ -43,13 +49,16 @@ test("app", async ({ context, page }) => {
4349
},
4450
],
4551
})
46-
await createWorkspace(page, template)
47-
await startAgent(page, token)
52+
const workspaceName = await createWorkspace(page, template)
53+
const agent = await startAgent(page, token)
4854

4955
// Wait for the web terminal to open in a new tab
5056
const pagePromise = context.waitForEvent("page")
5157
await page.getByText(appName).click()
5258
const app = await pagePromise
53-
await app.waitForLoadState("networkidle")
59+
await app.waitForLoadState("domcontentloaded")
5460
await app.getByText(appContent).isVisible()
61+
62+
await stopWorkspace(page, workspaceName)
63+
await stopAgent(agent)
5564
})

‎site/e2e/tests/gitAuth.spec.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ test("git auth device", async ({ page }) => {
4747
})
4848

4949
await page.goto(`/gitauth/${gitAuth.deviceProvider}`, {
50-
waitUntil: "networkidle",
50+
waitUntil: "domcontentloaded",
5151
})
5252
await page.getByText(device.user_code).isVisible()
5353
await sentPending.wait()
@@ -75,7 +75,7 @@ test("git auth web", async ({ baseURL, page }) => {
7575
)
7676
})
7777
await page.goto(`/gitauth/${gitAuth.webProvider}`, {
78-
waitUntil: "networkidle",
78+
waitUntil: "domcontentloaded",
7979
})
8080
// This endpoint doesn't have the installations URL set intentionally!
8181
await page.waitForSelector("text=You've authenticated with GitHub!")

‎site/e2e/tests/listTemplates.spec.ts

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
import { test, expect } from "@playwright/test"
2+
import { beforeCoderTest } from "../hooks"
3+
4+
test.beforeEach(async ({ page }) => await beforeCoderTest(page))
25

36
test("list templates", async ({ page, baseURL }) => {
4-
await page.goto(`${baseURL}/templates`, { waitUntil: "networkidle" })
7+
await page.goto(`${baseURL}/templates`, { waitUntil: "domcontentloaded" })
58
await expect(page).toHaveTitle("Templates - Coder")
69
})

‎site/e2e/tests/outdatedAgent.spec.ts

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ import {
66
downloadCoderVersion,
77
sshIntoWorkspace,
88
startAgentWithCommand,
9+
stopAgent,
10+
stopWorkspace,
911
} from "../helpers"
1012
import { beforeCoderTest } from "../hooks"
1113

@@ -32,11 +34,11 @@ test("ssh with agent " + agentVersion, async ({ page }) => {
3234
},
3335
],
3436
})
35-
const workspace = await createWorkspace(page, template)
37+
const workspaceName = await createWorkspace(page, template)
3638
const binaryPath = await downloadCoderVersion(agentVersion)
37-
await startAgentWithCommand(page, token, binaryPath)
39+
const agent = await startAgentWithCommand(page, token, binaryPath)
3840

39-
const client = await sshIntoWorkspace(page, workspace)
41+
const client = await sshIntoWorkspace(page, workspaceName)
4042
await new Promise<void>((resolve, reject) => {
4143
// We just exec a command to be certain the agent is running!
4244
client.exec("exit 0", (err, stream) => {
@@ -52,4 +54,7 @@ test("ssh with agent " + agentVersion, async ({ page }) => {
5254
})
5355
})
5456
})
57+
58+
await stopWorkspace(page, workspaceName)
59+
await stopAgent(agent, false)
5560
})

‎site/e2e/tests/outdatedCLI.spec.ts

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ import {
66
downloadCoderVersion,
77
sshIntoWorkspace,
88
startAgent,
9+
stopAgent,
10+
stopWorkspace,
911
} from "../helpers"
1012
import { beforeCoderTest } from "../hooks"
1113

@@ -32,11 +34,11 @@ test("ssh with client " + clientVersion, async ({ page }) => {
3234
},
3335
],
3436
})
35-
const workspace = await createWorkspace(page, template)
36-
await startAgent(page, token)
37+
const workspaceName = await createWorkspace(page, template)
38+
const agent = await startAgent(page, token)
3739
const binaryPath = await downloadCoderVersion(clientVersion)
3840

39-
const client = await sshIntoWorkspace(page, workspace, binaryPath)
41+
const client = await sshIntoWorkspace(page, workspaceName, binaryPath)
4042
await new Promise<void>((resolve, reject) => {
4143
// We just exec a command to be certain the agent is running!
4244
client.exec("exit 0", (err, stream) => {
@@ -52,4 +54,7 @@ test("ssh with client " + clientVersion, async ({ page }) => {
5254
})
5355
})
5456
})
57+
58+
await stopWorkspace(page, workspaceName)
59+
await stopAgent(agent)
5560
})

‎site/e2e/tests/webTerminal.spec.ts

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
import { test } from "@playwright/test"
2-
import { createTemplate, createWorkspace, startAgent } from "../helpers"
2+
import {
3+
createTemplate,
4+
createWorkspace,
5+
startAgent,
6+
stopAgent,
7+
} from "../helpers"
38
import { randomUUID } from "crypto"
49
import { beforeCoderTest } from "../hooks"
510

@@ -28,13 +33,13 @@ test("web terminal", async ({ context, page }) => {
2833
],
2934
})
3035
await createWorkspace(page, template)
31-
await startAgent(page, token)
36+
const agent = await startAgent(page, token)
3237

3338
// Wait for the web terminal to open in a new tab
3439
const pagePromise = context.waitForEvent("page")
3540
await page.getByTestId("terminal").click()
3641
const terminal = await pagePromise
37-
await terminal.waitForLoadState("networkidle")
42+
await terminal.waitForLoadState("domcontentloaded")
3843

3944
// Ensure that we can type in it
4045
await terminal.keyboard.type("echo hello")
@@ -50,4 +55,5 @@ test("web terminal", async ({ context, page }) => {
5055
}
5156
await new Promise((r) => setTimeout(r, 250))
5257
}
58+
await stopAgent(agent)
5359
})

0 commit comments

Comments
 (0)