Skip to content

test(site): add e2e tests for experiments #12940

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 20 commits into from
Apr 12, 2024
4 changes: 4 additions & 0 deletions site/e2e/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,7 @@ export const requireEnterpriseTests = Boolean(
process.env.CODER_E2E_REQUIRE_ENTERPRISE_TESTS,
);
export const enterpriseLicense = process.env.CODER_E2E_ENTERPRISE_LICENSE ?? "";

// Fake experiments to verify that site presents them as enabled.
export const e2eFakeExperiment1 = "e2e-fake-experiment-1";
export const e2eFakeExperiment2 = "e2e-fake-experiment-2";
14 changes: 12 additions & 2 deletions site/e2e/playwright.config.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,13 @@
import { defineConfig } from "@playwright/test";
import * as path from "path";
import { coderMain, coderPort, coderdPProfPort, gitAuth } from "./constants";
import {
coderMain,
coderPort,
coderdPProfPort,
e2eFakeExperiment1,
e2eFakeExperiment2,
gitAuth,
} from "./constants";

export const wsEndpoint = process.env.CODER_E2E_WS_ENDPOINT;

Expand All @@ -22,7 +29,7 @@ export default defineConfig({
testMatch: /.*\.spec\.ts/,
dependencies: ["testsSetup"],
use: { storageState },
timeout: 20_000,
timeout: 50_000,
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Otherwise:

2024-04-12T07:59:03.3989632Z ==> Errors
2024-04-12T07:59:03.3990075Z �[31mTest timeout of 20000ms exceeded.�[39m
2024-04-12T07:59:03.3991129Z /home/runner/actions-runner/_work/coder/coder/site/e2e/helpers.ts:357:
2024-04-12T07:59:03.3992071Z    at helpers.ts:357
2024-04-12T07:59:03.3992271Z 
2024-04-12T07:59:03.3992405Z   355 |   });
2024-04-12T07:59:03.3992661Z   356 |
2024-04-12T07:59:03.3993276Z > 357 |   await page.getByTestId("agent-status-ready").waitFor({ state: "visible" });
2024-04-12T07:59:03.3993951Z       |                                                ^
2024-04-12T07:59:03.3994334Z   358 |   return cp;
2024-04-12T07:59:03.3994667Z   359 | };
2024-04-12T07:59:03.3994999Z   360 |
2024-04-12T07:59:03.3995440Z Error: locator.waitFor: Test timeout of 20000ms exceeded.
2024-04-12T07:59:03.3995871Z Call log:
2024-04-12T07:59:03.3996400Z   �[2m- waiting for getByTestId('agent-status-ready') to be visible�[22m

link

},
],
reporter: [["./reporter.ts"]],
Expand Down Expand Up @@ -60,6 +67,8 @@ export default defineConfig({
.join(" "),
env: {
...process.env,
// Otherwise, the runner fails on Mac with: could not determine kind of name for C.uuid_string_t
CGO_ENABLED: "0",

// This is the test provider for git auth with devices!
CODER_GITAUTH_0_ID: gitAuth.deviceProvider,
Expand Down Expand Up @@ -101,6 +110,7 @@ export default defineConfig({
gitAuth.validatePath,
),
CODER_PPROF_ADDRESS: "127.0.0.1:" + coderdPProfPort,
CODER_EXPERIMENTS: e2eFakeExperiment1 + "," + e2eFakeExperiment2,
},
reuseExistingServer: false,
},
Expand Down
39 changes: 39 additions & 0 deletions site/e2e/tests/deployment/general.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import { expect, test } from "@playwright/test";
import * as API from "api/api";
import { setupApiCalls } from "../../api";
import { e2eFakeExperiment1, e2eFakeExperiment2 } from "../../constants";

test("experiments", async ({ page }) => {
await setupApiCalls(page);

// Load experiments from backend API
const availableExperiments = await API.getAvailableExperiments();

// Verify if the site lists the same experiments
await page.goto("/deployment/general", { waitUntil: "networkidle" });

const experimentsLocator = page.locator(
"div.options-table tr.option-experiments ul.option-array",
);
await expect(experimentsLocator).toBeVisible();

// Firstly, check if all enabled experiments are listed
expect(
experimentsLocator.locator(
`li.option-array-item-${e2eFakeExperiment1}.option-enabled`,
),
).toBeVisible;
expect(
experimentsLocator.locator(
`li.option-array-item-${e2eFakeExperiment2}.option-enabled`,
),
).toBeVisible;

// Secondly, check if available experiments are listed
for (const experiment of availableExperiments.safe) {
const experimentLocator = experimentsLocator.locator(
`li.option-array-item-${experiment}`,
);
await expect(experimentLocator).toBeVisible();
}
});
5 changes: 4 additions & 1 deletion site/src/pages/DeploySettingsPage/Option.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ export const OptionValue: FC<OptionValueProps> = (props) => {

if (typeof value === "object" && !Array.isArray(value)) {
return (
<ul css={{ listStyle: "none" }}>
<ul css={{ listStyle: "none" }} className="option-array">
{Object.entries(value)
.sort((a, b) => a[0].localeCompare(b[0]))
.map(([option, isEnabled]) => (
Expand All @@ -64,6 +64,9 @@ export const OptionValue: FC<OptionValueProps> = (props) => {
color: theme.palette.text.disabled,
},
]}
className={`option-array-item-${option} ${
isEnabled ? "option-enabled" : "option-disabled"
}`}
>
<div
css={{
Expand Down
4 changes: 2 additions & 2 deletions site/src/pages/DeploySettingsPage/OptionsTable.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ const OptionsTable: FC<OptionsTableProps> = ({ options, additionalValues }) => {
}

return (
<TableContainer>
<TableContainer className="options-table">
<Table
css={css`
& td {
Expand Down Expand Up @@ -57,7 +57,7 @@ const OptionsTable: FC<OptionsTableProps> = ({ options, additionalValues }) => {
return null;
}
return (
<TableRow key={option.flag}>
<TableRow key={option.flag} className={"option-" + option.flag}>
<TableCell>
<OptionName>{option.name}</OptionName>
<OptionDescription>{option.description}</OptionDescription>
Expand Down
Loading