diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 625e6a82673e1..e46aa7eb7383a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -382,8 +382,8 @@ jobs: touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile fi export TS_DEBUG_DISCO=true - gotestsum --junitfile="gotests.xml" --jsonfile="gotests.json" \ - --packages="./..." -- $PARALLEL_FLAG -short -failfast + gotestsum --junitfile="gotests.xml" --jsonfile="gotests.json" --rerun-fails=2 \ + --packages="./..." -- $PARALLEL_FLAG -short - name: Upload Test Cache uses: ./.github/actions/test-cache/upload @@ -436,6 +436,7 @@ jobs: TS_DEBUG_DISCO: "true" LC_CTYPE: "en_US.UTF-8" LC_ALL: "en_US.UTF-8" + TEST_RETRIES: 2 shell: bash run: | # By default Go will use the number of logical CPUs, which @@ -499,6 +500,7 @@ jobs: TS_DEBUG_DISCO: "true" LC_CTYPE: "en_US.UTF-8" LC_ALL: "en_US.UTF-8" + TEST_RETRIES: 2 shell: bash run: | # By default Go will use the number of logical CPUs, which @@ -560,6 +562,7 @@ jobs: env: POSTGRES_VERSION: "16" TS_DEBUG_DISCO: "true" + TEST_RETRIES: 2 run: | make test-postgres @@ -784,6 +787,7 @@ jobs: if: ${{ !matrix.variant.premium }} env: DEBUG: pw:api + CODER_E2E_TEST_RETRIES: 2 working-directory: site # Run all of the tests with a premium license @@ -793,6 +797,7 @@ jobs: DEBUG: pw:api CODER_E2E_LICENSE: ${{ secrets.CODER_E2E_LICENSE }} CODER_E2E_REQUIRE_PREMIUM_TESTS: "1" + CODER_E2E_TEST_RETRIES: 2 working-directory: site - name: Upload Playwright Failed Tests diff --git a/Makefile b/Makefile index f96c8ab957442..0b8cefbab0663 100644 --- a/Makefile +++ b/Makefile @@ -875,12 +875,19 @@ provisioner/terraform/testdata/version: fi .PHONY: provisioner/terraform/testdata/version +# Set the retry flags if TEST_RETRIES is set +ifdef TEST_RETRIES +GOTESTSUM_RETRY_FLAGS := --rerun-fails=$(TEST_RETRIES) +else +GOTESTSUM_RETRY_FLAGS := +endif + test: - $(GIT_FLAGS) gotestsum --format standard-quiet -- -v -short -count=1 ./... $(if $(RUN),-run $(RUN)) + $(GIT_FLAGS) gotestsum --format standard-quiet $(GOTESTSUM_RETRY_FLAGS) --packages="./..." -- -v -short -count=1 $(if $(RUN),-run $(RUN)) .PHONY: test test-cli: - $(GIT_FLAGS) gotestsum --format standard-quiet -- -v -short -count=1 ./cli/... + $(GIT_FLAGS) gotestsum --format standard-quiet $(GOTESTSUM_RETRY_FLAGS) --packages="./cli/..." -- -v -short -count=1 .PHONY: test-cli # sqlc-cloud-is-setup will fail if no SQLc auth token is set. Use this as a @@ -919,9 +926,9 @@ test-postgres: test-postgres-docker $(GIT_FLAGS) DB=ci gotestsum \ --junitfile="gotests.xml" \ --jsonfile="gotests.json" \ + $(GOTESTSUM_RETRY_FLAGS) \ --packages="./..." -- \ -timeout=20m \ - -failfast \ -count=1 .PHONY: test-postgres diff --git a/site/e2e/playwright.config.ts b/site/e2e/playwright.config.ts index 762b7f0158dba..436af99240493 100644 --- a/site/e2e/playwright.config.ts +++ b/site/e2e/playwright.config.ts @@ -10,12 +10,30 @@ import { } from "./constants"; export const wsEndpoint = process.env.CODER_E2E_WS_ENDPOINT; +export const retries = (() => { + if (process.env.CODER_E2E_TEST_RETRIES === undefined) { + return undefined; + } + const count = Number.parseInt(process.env.CODER_E2E_TEST_RETRIES, 10); + if (Number.isNaN(count)) { + throw new Error( + `CODER_E2E_TEST_RETRIES is not a number: ${process.env.CODER_E2E_TEST_RETRIES}`, + ); + } + if (count < 0) { + throw new Error( + `CODER_E2E_TEST_RETRIES is less than 0: ${process.env.CODER_E2E_TEST_RETRIES}`, + ); + } + return count; +})(); const localURL = (port: number, path: string): string => { return `http://localhost:${port}${path}`; }; export default defineConfig({ + retries, globalSetup: require.resolve("./setup/preflight"), projects: [ {