From d7521ba7a5f4c6a948cb3e0bb6474ff4c44cf32b Mon Sep 17 00:00:00 2001 From: McKayla Washburn Date: Thu, 4 Apr 2024 20:41:03 +0000 Subject: [PATCH 1/3] test: verify that enterprise tests are being run --- .github/workflows/ci.yaml | 1 + Makefile | 6 +- site/e2e/constants.ts | 3 + site/e2e/global.setup.ts | 9 ++- site/e2e/helpers.ts | 5 ++ site/e2e/reporter.ts | 71 ++++++++++++------- site/e2e/tests/enterprise.spec.ts | 10 +++ .../WorkspacePage/WorkspacePage.test.tsx | 6 +- 8 files changed, 78 insertions(+), 33 deletions(-) create mode 100644 site/e2e/tests/enterprise.spec.ts diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c3541a24937b2..c35f564dcd5ea 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -483,6 +483,7 @@ jobs: env: DEBUG: pw:api CODER_E2E_ENTERPRISE_LICENSE: ${{ secrets.CODER_E2E_ENTERPRISE_LICENSE }} + CODER_E2E_REQUIRE_ENTERPRISE_TESTS: "1" working-directory: site - name: Upload Playwright Failed Tests diff --git a/Makefile b/Makefile index 84a97323cd442..e588279384baa 100644 --- a/Makefile +++ b/Makefile @@ -382,9 +382,9 @@ install: build/coder_$(VERSION)_$(GOOS)_$(GOARCH)$(GOOS_BIN_EXT) cp "$<" "$$output_file" .PHONY: install -BOLD := $(shell tput bold) -GREEN := $(shell tput setaf 2) -RESET := $(shell tput sgr0) +BOLD := $(shell tput bold 2>/dev/null) +GREEN := $(shell tput setaf 2 2>/dev/null) +RESET := $(shell tput sgr0 2>/dev/null) fmt: fmt/eslint fmt/prettier fmt/terraform fmt/shfmt fmt/go .PHONY: fmt diff --git a/site/e2e/constants.ts b/site/e2e/constants.ts index 8b4fbe50d5a7d..351af63be249c 100644 --- a/site/e2e/constants.ts +++ b/site/e2e/constants.ts @@ -33,4 +33,7 @@ export const gitAuth = { installationsPath: "/installations", }; +export const requireEnterpriseTests = Boolean( + process.env.CODER_E2E_REQUIRE_ENTERPRISE_TESTS, +); export const enterpriseLicense = process.env.CODER_E2E_ENTERPRISE_LICENSE ?? ""; diff --git a/site/e2e/global.setup.ts b/site/e2e/global.setup.ts index 9a9d4d026fa83..b4f92c423bdab 100644 --- a/site/e2e/global.setup.ts +++ b/site/e2e/global.setup.ts @@ -1,4 +1,4 @@ -import { test, expect } from "@playwright/test"; +import { expect, test } from "@playwright/test"; import { Language } from "pages/CreateUserPage/CreateUserForm"; import * as constants from "./constants"; import { storageState } from "./playwright.config"; @@ -18,7 +18,12 @@ test("setup deployment", async ({ page }) => { await page.getByTestId("button-select-template").isVisible(); // Setup license - if (constants.enterpriseLicense) { + if (constants.requireEnterpriseTests || constants.enterpriseLicense) { + // Make sure that we have something that looks like a real license + expect(constants.enterpriseLicense).toBeTruthy(); + expect(constants.enterpriseLicense.length).toBeGreaterThan(92); // the signature alone should be this long + expect(constants.enterpriseLicense.split(".").length).toBe(3); // otherwise it's invalid + await page.goto("/deployment/licenses", { waitUntil: "domcontentloaded" }); await page.getByText("Add a license").click(); diff --git a/site/e2e/helpers.ts b/site/e2e/helpers.ts index 519a8968f89f3..e7acb7ff616ad 100644 --- a/site/e2e/helpers.ts +++ b/site/e2e/helpers.ts @@ -18,6 +18,7 @@ import { coderPort, enterpriseLicense, prometheusPort, + requireEnterpriseTests, } from "./constants"; import { Agent, @@ -33,6 +34,10 @@ import { // requiresEnterpriseLicense will skip the test if we're not running with an enterprise license export function requiresEnterpriseLicense() { + if (requireEnterpriseTests) { + return; + } + test.skip(!enterpriseLicense); } diff --git a/site/e2e/reporter.ts b/site/e2e/reporter.ts index 9b4eaabd5ba1b..fe214d8aed41d 100644 --- a/site/e2e/reporter.ts +++ b/site/e2e/reporter.ts @@ -11,12 +11,13 @@ import type { import axios from "axios"; import * as fs from "fs/promises"; import type { Writable } from "stream"; -import { coderdPProfPort } from "./constants"; +import { coderdPProfPort, enterpriseLicense } from "./constants"; class CoderReporter implements Reporter { config: FullConfig | null = null; testOutput = new Map>(); passedCount = 0; + skippedCount = 0; failedTests: TestCase[] = []; timedOutTests: TestCase[] = []; @@ -31,45 +32,56 @@ class CoderReporter implements Reporter { } onStdOut(chunk: string, test?: TestCase, _?: TestResult): void { - for (const line of filteredServerLogLines(chunk)) { - console.log(`[stdout] ${line}`); - } + // If there's no associated test, just print it now if (!test) { + for (const line of logLines(chunk)) { + console.log(`[stdout] ${line}`); + } return; } + // Will be printed if the test fails this.testOutput.get(test.id)!.push([process.stdout, chunk]); } onStdErr(chunk: string, test?: TestCase, _?: TestResult): void { - for (const line of filteredServerLogLines(chunk)) { - console.error(`[stderr] ${line}`); - } + // If there's no associated test, just print it now if (!test) { + for (const line of logLines(chunk)) { + console.error(`[stderr] ${line}`); + } return; } + // Will be printed if the test fails this.testOutput.get(test.id)!.push([process.stderr, chunk]); } async onTestEnd(test: TestCase, result: TestResult) { - console.log(`==> Finished test ${test.title}: ${result.status}`); + try { + if (test.expectedStatus === "skipped") { + console.log(`==> Skipping test ${test.title}`); + this.skippedCount++; + return; + } - if (result.status === "passed") { - this.passedCount++; - } + console.log(`==> Finished test ${test.title}: ${result.status}`); - if (result.status === "failed") { - this.failedTests.push(test); - } + if (result.status === "passed") { + this.passedCount++; + return; + } - if (result.status === "timedOut") { - this.timedOutTests.push(test); - } + if (result.status === "failed") { + this.failedTests.push(test); + } + + if (result.status === "timedOut") { + this.timedOutTests.push(test); + } - const fsTestTitle = test.title.replaceAll(" ", "-"); - const outputFile = `test-results/debug-pprof-goroutine-${fsTestTitle}.txt`; - await exportDebugPprof(outputFile); + const fsTestTitle = test.title.replaceAll(" ", "-"); + const outputFile = `test-results/debug-pprof-goroutine-${fsTestTitle}.txt`; + await exportDebugPprof(outputFile); - if (result.status !== "passed") { console.log(`Data from pprof has been saved to ${outputFile}`); console.log("==> Output"); const output = this.testOutput.get(test.id)!; @@ -90,13 +102,22 @@ class CoderReporter implements Reporter { console.log(attachment); } } + } finally { + this.testOutput.delete(test.id); } - this.testOutput.delete(test.id); } onEnd(result: FullResult) { console.log(`==> Tests ${result.status}`); + if (!enterpriseLicense) { + console.log( + "==> Enterprise tests were skipped, because no license was provided", + ); + } console.log(`${this.passedCount} passed`); + if (this.skippedCount > 0) { + console.log(`${this.skippedCount} skipped`); + } if (this.failedTests.length > 0) { console.log(`${this.failedTests.length} failed`); for (const test of this.failedTests) { @@ -112,11 +133,7 @@ class CoderReporter implements Reporter { } } -const shouldPrintLine = (line: string) => - [" error=EOF", "coderd: audit_log"].every((noise) => !line.includes(noise)); - -const filteredServerLogLines = (chunk: string): string[] => - chunk.trimEnd().split("\n").filter(shouldPrintLine); +const logLines = (chunk: string): string[] => chunk.trimEnd().split("\n"); const exportDebugPprof = async (outputFile: string) => { const response = await axios.get( diff --git a/site/e2e/tests/enterprise.spec.ts b/site/e2e/tests/enterprise.spec.ts new file mode 100644 index 0000000000000..4758d43ae1802 --- /dev/null +++ b/site/e2e/tests/enterprise.spec.ts @@ -0,0 +1,10 @@ +import { expect, test } from "@playwright/test"; +import { requiresEnterpriseLicense } from "../helpers"; + +test("license was added successfully", async ({ page }) => { + requiresEnterpriseLicense(); + + await page.goto("/deployment/licenses", { waitUntil: "domcontentloaded" }); + const license = page.locator(".MuiPaper-root", { hasText: "#1" }); + await expect(license).toBeVisible(); +}); diff --git a/site/src/pages/WorkspacePage/WorkspacePage.test.tsx b/site/src/pages/WorkspacePage/WorkspacePage.test.tsx index 0a69834992638..a3bf16188db63 100644 --- a/site/src/pages/WorkspacePage/WorkspacePage.test.tsx +++ b/site/src/pages/WorkspacePage/WorkspacePage.test.tsx @@ -18,7 +18,10 @@ import { MockDeploymentConfig, MockWorkspaceBuildDelete, } from "testHelpers/entities"; -import { renderWithAuth } from "testHelpers/renderHelpers"; +import { + renderWithAuth, + waitForLoaderToBeRemoved, +} from "testHelpers/renderHelpers"; import { server } from "testHelpers/server"; import { WorkspacePage } from "./WorkspacePage"; @@ -42,6 +45,7 @@ const renderWorkspacePage = async (workspace: Workspace) => { path: "/:username/:workspace", }); + await waitForLoaderToBeRemoved(); await screen.findByText(workspace.name); }; From 80516fdaa0cc3bd356e044fb8a43825e92afae1b Mon Sep 17 00:00:00 2001 From: McKayla Washburn Date: Thu, 4 Apr 2024 20:53:59 +0000 Subject: [PATCH 2/3] never mind --- site/src/pages/WorkspacePage/WorkspacePage.test.tsx | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/site/src/pages/WorkspacePage/WorkspacePage.test.tsx b/site/src/pages/WorkspacePage/WorkspacePage.test.tsx index a3bf16188db63..0a69834992638 100644 --- a/site/src/pages/WorkspacePage/WorkspacePage.test.tsx +++ b/site/src/pages/WorkspacePage/WorkspacePage.test.tsx @@ -18,10 +18,7 @@ import { MockDeploymentConfig, MockWorkspaceBuildDelete, } from "testHelpers/entities"; -import { - renderWithAuth, - waitForLoaderToBeRemoved, -} from "testHelpers/renderHelpers"; +import { renderWithAuth } from "testHelpers/renderHelpers"; import { server } from "testHelpers/server"; import { WorkspacePage } from "./WorkspacePage"; @@ -45,7 +42,6 @@ const renderWorkspacePage = async (workspace: Workspace) => { path: "/:username/:workspace", }); - await waitForLoaderToBeRemoved(); await screen.findByText(workspace.name); }; From 322a1403f2846f51af217e4d203bd305b4922974 Mon Sep 17 00:00:00 2001 From: McKayla Washburn Date: Fri, 5 Apr 2024 17:33:49 +0000 Subject: [PATCH 3/3] actually, leave it out for now --- .github/workflows/ci.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 09dcc22b24a25..b255a0e4429c2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -478,15 +478,6 @@ jobs: DEBUG: pw:api working-directory: site - # Run all of the tests with an enterprise license - - run: pnpm playwright:test --forbid-only --workers 1 - env: - DEBUG: pw:api - CODER_E2E_ENTERPRISE_LICENSE: ${{ secrets.CODER_E2E_ENTERPRISE_LICENSE }} - CODER_E2E_REQUIRE_ENTERPRISE_TESTS: "1" - working-directory: site - continue-on-error: true - - name: Upload Playwright Failed Tests if: always() && github.actor != 'dependabot[bot]' && runner.os == 'Linux' && !github.event.pull_request.head.repo.fork uses: actions/upload-artifact@v4