Skip to content

test: improve E2E framework #9469

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Sep 4, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Use pkill
  • Loading branch information
mtojek committed Sep 1, 2023
commit fd5850a7030b7498a45e09dc7466d5003ba3fc86
44 changes: 39 additions & 5 deletions site/e2e/helpers.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { expect, Page } from "@playwright/test"
import { spawn } from "child_process"
import { ChildProcess, exec, spawn } from "child_process"
import { randomUUID } from "crypto"
import path from "path"
import express from "express"
Expand All @@ -19,6 +19,7 @@ import { port } from "./playwright.config"
import * as ssh from "ssh2"
import { Duplex } from "stream"
import { WorkspaceBuildParameter } from "api/typesGenerated"
import axios from "axios"

// createWorkspace creates a workspace for a template.
// It does not wait for it to be running, but it does navigate to the page.
Expand Down Expand Up @@ -229,7 +230,10 @@ export const buildWorkspaceWithParameters = async (

// startAgent runs the coder agent with the provided token.
// It awaits the agent to be ready before returning.
export const startAgent = async (page: Page, token: string): Promise<void> => {
export const startAgent = async (
page: Page,
token: string,
): Promise<ChildProcess> => {
return startAgentWithCommand(page, token, "go", "run", coderMainPath())
}

Expand Down Expand Up @@ -308,14 +312,14 @@ export const startAgentWithCommand = async (
token: string,
command: string,
...args: string[]
): Promise<void> => {
): Promise<ChildProcess> => {
const cp = spawn(command, [...args, "agent", "--no-reap"], {
env: {
...process.env,
CODER_AGENT_URL: "http://localhost:" + port,
CODER_AGENT_TOKEN: token,
CODER_AGENT_PPROF_ADDRESS: "127.0.0.1:2114",
CODER_AGENT_PROMETHEUS_ADDRESS: "127.0.0.1:6061",
CODER_AGENT_PPROF_ADDRESS: "127.0.0.1:6061",
CODER_AGENT_PROMETHEUS_ADDRESS: "127.0.0.1:2114",
},
})
cp.stdout.on("data", (data: Buffer) => {
Expand All @@ -332,6 +336,36 @@ export const startAgentWithCommand = async (
})

await page.getByTestId("agent-status-ready").waitFor({ state: "visible" })
return cp
}

export const stopAgent = async (cp: ChildProcess) => {
exec(`pkill -P ${cp.pid}`, (error) => {
if (error) {
throw new Error(`exec error: ${JSON.stringify(error)}`)
}
})
await waitUntilUrlIsNotResponding("http://127.0.0.1:2114")
}

const waitUntilUrlIsNotResponding = async (url: string) => {
const maxRetries = 30
const retryIntervalMs = 1000
let retries = 0

while (retries < maxRetries) {
try {
await axios.get(url)
} catch (error) {
return
}

retries++
await new Promise((resolve) => setTimeout(resolve, retryIntervalMs))
}
throw new Error(
`URL ${url} is still responding after ${maxRetries * retryIntervalMs}ms`,
)
}

const coderMainPath = (): string => {
Expand Down
4 changes: 3 additions & 1 deletion site/e2e/tests/app.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {
createTemplate,
createWorkspace,
startAgent,
stopAgent,
stopWorkspace,
} from "../helpers"
import { beforeCoderTest } from "../hooks"
Expand Down Expand Up @@ -49,7 +50,7 @@ test("app", async ({ context, page }) => {
],
})
const workspaceName = await createWorkspace(page, template)
await startAgent(page, token)
const agent = await startAgent(page, token)

// Wait for the web terminal to open in a new tab
const pagePromise = context.waitForEvent("page")
Expand All @@ -59,4 +60,5 @@ test("app", async ({ context, page }) => {
await app.getByText(appContent).isVisible()

await stopWorkspace(page, workspaceName)
await stopAgent(agent)
})
3 changes: 3 additions & 0 deletions site/e2e/tests/listTemplates.spec.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import { test, expect } from "@playwright/test"
import { beforeCoderTest } from "../hooks"

test.beforeEach(async ({ page }) => await beforeCoderTest(page))

test("list templates", async ({ page, baseURL }) => {
await page.goto(`${baseURL}/templates`, { waitUntil: "domcontentloaded" })
Expand Down
4 changes: 3 additions & 1 deletion site/e2e/tests/outdatedAgent.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
downloadCoderVersion,
sshIntoWorkspace,
startAgentWithCommand,
stopAgent,
stopWorkspace,
} from "../helpers"
import { beforeCoderTest } from "../hooks"
Expand Down Expand Up @@ -35,7 +36,7 @@ test("ssh with agent " + agentVersion, async ({ page }) => {
})
const workspaceName = await createWorkspace(page, template)
const binaryPath = await downloadCoderVersion(agentVersion)
await startAgentWithCommand(page, token, binaryPath)
const agent = await startAgentWithCommand(page, token, binaryPath)

const client = await sshIntoWorkspace(page, workspaceName)
await new Promise<void>((resolve, reject) => {
Expand All @@ -55,4 +56,5 @@ test("ssh with agent " + agentVersion, async ({ page }) => {
})

await stopWorkspace(page, workspaceName)
await stopAgent(agent)
})
4 changes: 3 additions & 1 deletion site/e2e/tests/outdatedCLI.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
downloadCoderVersion,
sshIntoWorkspace,
startAgent,
stopAgent,
stopWorkspace,
} from "../helpers"
import { beforeCoderTest } from "../hooks"
Expand Down Expand Up @@ -34,7 +35,7 @@ test("ssh with client " + clientVersion, async ({ page }) => {
],
})
const workspaceName = await createWorkspace(page, template)
await startAgent(page, token)
const agent = await startAgent(page, token)
const binaryPath = await downloadCoderVersion(clientVersion)

const client = await sshIntoWorkspace(page, workspaceName, binaryPath)
Expand All @@ -55,4 +56,5 @@ test("ssh with client " + clientVersion, async ({ page }) => {
})

await stopWorkspace(page, workspaceName)
await stopAgent(agent)
})
10 changes: 8 additions & 2 deletions site/e2e/tests/webTerminal.spec.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
import { test } from "@playwright/test"
import { createTemplate, createWorkspace, startAgent } from "../helpers"
import {
createTemplate,
createWorkspace,
startAgent,
stopAgent,
} from "../helpers"
import { randomUUID } from "crypto"
import { beforeCoderTest } from "../hooks"

Expand Down Expand Up @@ -28,7 +33,7 @@ test("web terminal", async ({ context, page }) => {
],
})
await createWorkspace(page, template)
await startAgent(page, token)
const agent = await startAgent(page, token)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We still have the unfortunate problem that any error in a test will terminate execution and the agent won't be cleaned up. Can we somehow make it a post-test cleanup hook that always runs and checks if an agent is running?

This isn't a big problem per-se but can cause trouble when running locally -> suddenly all tests fail after one failure. And also cause noise in test output.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was trying to implement an easy way but failed. The clue is to pass the PID of the process/group of process to the test.afterEach. Something like "clean used resources", but not necessarily re-discover them.


// Wait for the web terminal to open in a new tab
const pagePromise = context.waitForEvent("page")
Expand All @@ -50,4 +55,5 @@ test("web terminal", async ({ context, page }) => {
}
await new Promise((r) => setTimeout(r, 250))
}
await stopAgent(agent)
})