From 19cf57b559ad064e6a0504b832e7d69249c2fa07 Mon Sep 17 00:00:00 2001 From: Bryan Phelps Date: Mon, 21 Feb 2022 18:26:11 +0000 Subject: [PATCH 01/15] refactor: Add nightly test stability workflow --- .github/workflows/coder-test-stability.yaml | 88 +++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 .github/workflows/coder-test-stability.yaml diff --git a/.github/workflows/coder-test-stability.yaml b/.github/workflows/coder-test-stability.yaml new file mode 100644 index 0000000000000..d80c389560c58 --- /dev/null +++ b/.github/workflows/coder-test-stability.yaml @@ -0,0 +1,88 @@ +# This workflow (aka The Gauntlet) is a high-iteration run of our tests, +# used to evaluate stability and shake out intermittent failures. +name: coder-test-stability +on: + schedule: + # Run everyday around midnight Central. + - cron: "0 6 * * *" + + pull_request: + branches: + - master + paths: + - .github/workflows/coder-test-stability.yaml + workflow_dispatch: + +# Cancel in-progress runs for pull requests when developers push +# additional changes, and serialize builds in branches. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-concurrency-to-cancel-any-in-progress-job-or-run +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + coder-test-stability: + name: "test/go/stability" + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-2022 + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-go@v2 + with: + go-version: "^1.17" + + - uses: actions/cache@v2 + with: + # Go mod cache, Linux build cache, Mac build cache, Windows build cache + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ matrix.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ matrix.os }}-go- + + - run: go install gotest.tools/gotestsum@latest + + - uses: hashicorp/setup-terraform@v1 + with: + terraform_version: 1.1.2 + terraform_wrapper: false + + - name: Test with Mock Database + shell: bash + env: + GOCOUNT: 25 + GOMAXPROCS: ${{ runner.os == 'Windows' && 1 || 2 }} + run: gotestsum --junitfile="gotests.xml" --packages="./..." -- + -covermode=atomic -coverprofile="gotests.coverage" + -timeout=3m -count=$GOCOUNT -race -short -failfast + + - name: Upload DataDog Trace + if: (success() || failure()) && github.actor != 'dependabot[bot]' + env: + DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} + DD_DATABASE: fake + GIT_COMMIT_MESSAGE: stability-${{ github.event.head_commit.message }} + run: go run scripts/datadog-cireport/main.go gotests.xml + + - name: Test with PostgreSQL Database + if: runner.os == 'Linux' + run: DB=true gotestsum --junitfile="gotests.xml" --packages="./..." -- + -covermode=atomic -coverprofile="gotests.coverage" -timeout=3m + -count=25 -race -parallel=2 -failfast + + - name: Upload DataDog Trace + if: (success() || failure()) && github.actor != 'dependabot[bot]' && runner.os == 'Linux' + env: + DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} + DD_DATABASE: postgresql + GIT_COMMIT_MESSAGE: stability-${{ github.event.head_commit.message }} + run: go run scripts/datadog-cireport/main.go gotests.xml \ No newline at end of file From 18dae0ed20e7a454493290a847767d66df9ab651 Mon Sep 17 00:00:00 2001 From: Bryan Phelps Date: Mon, 21 Feb 2022 18:28:41 +0000 Subject: [PATCH 02/15] Fix branch name --- .github/workflows/coder-test-stability.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coder-test-stability.yaml b/.github/workflows/coder-test-stability.yaml index d80c389560c58..12d7059383cff 100644 --- a/.github/workflows/coder-test-stability.yaml +++ b/.github/workflows/coder-test-stability.yaml @@ -8,7 +8,7 @@ on: pull_request: branches: - - master + - main paths: - .github/workflows/coder-test-stability.yaml workflow_dispatch: From 41fca23fbdc65bcd102d111e36cac113221f06c1 Mon Sep 17 00:00:00 2001 From: Bryan Phelps Date: Mon, 21 Feb 2022 18:30:24 +0000 Subject: [PATCH 03/15] Temporarily switch branch --- .github/workflows/coder-test-stability.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/coder-test-stability.yaml b/.github/workflows/coder-test-stability.yaml index 12d7059383cff..1bce323d37023 100644 --- a/.github/workflows/coder-test-stability.yaml +++ b/.github/workflows/coder-test-stability.yaml @@ -8,7 +8,8 @@ on: pull_request: branches: - - main + # - main + - "*" paths: - .github/workflows/coder-test-stability.yaml workflow_dispatch: From a6d35923e69f37a964a582ea75765858321071e1 Mon Sep 17 00:00:00 2001 From: Bryan Phelps Date: Mon, 21 Feb 2022 18:30:50 +0000 Subject: [PATCH 04/15] Add TODO comment --- .github/workflows/coder-test-stability.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/coder-test-stability.yaml b/.github/workflows/coder-test-stability.yaml index 1bce323d37023..a9661ad96e2a7 100644 --- a/.github/workflows/coder-test-stability.yaml +++ b/.github/workflows/coder-test-stability.yaml @@ -8,6 +8,7 @@ on: pull_request: branches: + # TODO: Remove before merge # - main - "*" paths: From 4afef6111732ba2b7c500679ce727b0d50770fd1 Mon Sep 17 00:00:00 2001 From: Bryan Phelps Date: Wed, 23 Feb 2022 04:53:46 +0000 Subject: [PATCH 05/15] Add os name to stability job --- .github/workflows/coder-test-stability.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coder-test-stability.yaml b/.github/workflows/coder-test-stability.yaml index a9661ad96e2a7..9681176de6c85 100644 --- a/.github/workflows/coder-test-stability.yaml +++ b/.github/workflows/coder-test-stability.yaml @@ -24,7 +24,7 @@ concurrency: jobs: coder-test-stability: - name: "test/go/stability" + name: "test/go/stability/${{ matrix.os }}" runs-on: ${{ matrix.os }} strategy: matrix: From 5b867a1004e4b5073048e653cace20c3c236afca Mon Sep 17 00:00:00 2001 From: Bryan Phelps Date: Wed, 23 Feb 2022 04:58:52 +0000 Subject: [PATCH 06/15] Add input parameter for iterationCount --- .github/workflows/coder-test-stability.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coder-test-stability.yaml b/.github/workflows/coder-test-stability.yaml index 9681176de6c85..3805340e1b2bf 100644 --- a/.github/workflows/coder-test-stability.yaml +++ b/.github/workflows/coder-test-stability.yaml @@ -14,6 +14,11 @@ on: paths: - .github/workflows/coder-test-stability.yaml workflow_dispatch: + inputs: + iterationCount: + description: 'Iteration Count' + required: false + default: '25' # Cancel in-progress runs for pull requests when developers push # additional changes, and serialize builds in branches. @@ -61,7 +66,7 @@ jobs: - name: Test with Mock Database shell: bash env: - GOCOUNT: 25 + GOCOUNT: ${{ github.event.inputs.logLevel }} GOMAXPROCS: ${{ runner.os == 'Windows' && 1 || 2 }} run: gotestsum --junitfile="gotests.xml" --packages="./..." -- -covermode=atomic -coverprofile="gotests.coverage" @@ -79,7 +84,7 @@ jobs: if: runner.os == 'Linux' run: DB=true gotestsum --junitfile="gotests.xml" --packages="./..." -- -covermode=atomic -coverprofile="gotests.coverage" -timeout=3m - -count=25 -race -parallel=2 -failfast + -count=${{ github.event.inputs.logLevel }} -race -parallel=2 -failfast - name: Upload DataDog Trace if: (success() || failure()) && github.actor != 'dependabot[bot]' && runner.os == 'Linux' From 33be0c80ff2cd5a81214b15f34a1283d56626750 Mon Sep 17 00:00:00 2001 From: Bryan Phelps Date: Wed, 23 Feb 2022 05:01:52 +0000 Subject: [PATCH 07/15] Fix copy-paste error --- .github/workflows/coder-test-stability.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coder-test-stability.yaml b/.github/workflows/coder-test-stability.yaml index 3805340e1b2bf..a898349d5c58c 100644 --- a/.github/workflows/coder-test-stability.yaml +++ b/.github/workflows/coder-test-stability.yaml @@ -66,7 +66,7 @@ jobs: - name: Test with Mock Database shell: bash env: - GOCOUNT: ${{ github.event.inputs.logLevel }} + GOCOUNT: ${{ github.event.inputs.iterationCount }} GOMAXPROCS: ${{ runner.os == 'Windows' && 1 || 2 }} run: gotestsum --junitfile="gotests.xml" --packages="./..." -- -covermode=atomic -coverprofile="gotests.coverage" @@ -84,7 +84,7 @@ jobs: if: runner.os == 'Linux' run: DB=true gotestsum --junitfile="gotests.xml" --packages="./..." -- -covermode=atomic -coverprofile="gotests.coverage" -timeout=3m - -count=${{ github.event.inputs.logLevel }} -race -parallel=2 -failfast + -count=${{ github.event.inputs.iterationCount }} -race -parallel=2 -failfast - name: Upload DataDog Trace if: (success() || failure()) && github.actor != 'dependabot[bot]' && runner.os == 'Linux' From a0c1800c628074435509087fa41586548380112a Mon Sep 17 00:00:00 2001 From: Bryan Phelps Date: Wed, 23 Feb 2022 05:07:59 +0000 Subject: [PATCH 08/15] Add default, since inputs only get populated on workflow dispatches --- .github/workflows/coder-test-stability.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coder-test-stability.yaml b/.github/workflows/coder-test-stability.yaml index a898349d5c58c..c0be21398d329 100644 --- a/.github/workflows/coder-test-stability.yaml +++ b/.github/workflows/coder-test-stability.yaml @@ -66,7 +66,7 @@ jobs: - name: Test with Mock Database shell: bash env: - GOCOUNT: ${{ github.event.inputs.iterationCount }} + GOCOUNT: ${{ github.event.inputs.iterationCount || 25 }} GOMAXPROCS: ${{ runner.os == 'Windows' && 1 || 2 }} run: gotestsum --junitfile="gotests.xml" --packages="./..." -- -covermode=atomic -coverprofile="gotests.coverage" @@ -82,9 +82,11 @@ jobs: - name: Test with PostgreSQL Database if: runner.os == 'Linux' + env: + GOCOUNT: ${{ github.event.inputs.iterationCount || 25 }} run: DB=true gotestsum --junitfile="gotests.xml" --packages="./..." -- -covermode=atomic -coverprofile="gotests.coverage" -timeout=3m - -count=${{ github.event.inputs.iterationCount }} -race -parallel=2 -failfast + -count=$GOCOUNT -race -parallel=2 -failfast - name: Upload DataDog Trace if: (success() || failure()) && github.actor != 'dependabot[bot]' && runner.os == 'Linux' From e5c4e1870f0d3f294ebf48152b00e3532857df9f Mon Sep 17 00:00:00 2001 From: Bryan Phelps Date: Wed, 23 Feb 2022 05:14:15 +0000 Subject: [PATCH 09/15] Add multiple parallel instances --- .github/workflows/coder-test-stability.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/coder-test-stability.yaml b/.github/workflows/coder-test-stability.yaml index c0be21398d329..4f690cfb93a0a 100644 --- a/.github/workflows/coder-test-stability.yaml +++ b/.github/workflows/coder-test-stability.yaml @@ -18,7 +18,7 @@ on: iterationCount: description: 'Iteration Count' required: false - default: '25' + default: '15' # Cancel in-progress runs for pull requests when developers push # additional changes, and serialize builds in branches. @@ -29,7 +29,7 @@ concurrency: jobs: coder-test-stability: - name: "test/go/stability/${{ matrix.os }}" + name: "test/go/stability/${{ matrix.os }}/${{ matrix.instance }}" runs-on: ${{ matrix.os }} strategy: matrix: @@ -37,6 +37,9 @@ jobs: - ubuntu-latest - macos-latest - windows-2022 + instance: + - 1 + - 2 steps: - uses: actions/checkout@v2 @@ -66,7 +69,7 @@ jobs: - name: Test with Mock Database shell: bash env: - GOCOUNT: ${{ github.event.inputs.iterationCount || 25 }} + GOCOUNT: ${{ github.event.inputs.iterationCount || 15 }} GOMAXPROCS: ${{ runner.os == 'Windows' && 1 || 2 }} run: gotestsum --junitfile="gotests.xml" --packages="./..." -- -covermode=atomic -coverprofile="gotests.coverage" @@ -83,7 +86,7 @@ jobs: - name: Test with PostgreSQL Database if: runner.os == 'Linux' env: - GOCOUNT: ${{ github.event.inputs.iterationCount || 25 }} + GOCOUNT: ${{ github.event.inputs.iterationCount || 15 }} run: DB=true gotestsum --junitfile="gotests.xml" --packages="./..." -- -covermode=atomic -coverprofile="gotests.coverage" -timeout=3m -count=$GOCOUNT -race -parallel=2 -failfast From 34a1049ff2b2e150b423604113f1f74e122b18c5 Mon Sep 17 00:00:00 2001 From: Bryan Phelps Date: Wed, 23 Feb 2022 05:46:31 +0000 Subject: [PATCH 10/15] Bump up test timeout --- .github/workflows/coder-test-stability.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coder-test-stability.yaml b/.github/workflows/coder-test-stability.yaml index 4f690cfb93a0a..a67d335c5dec7 100644 --- a/.github/workflows/coder-test-stability.yaml +++ b/.github/workflows/coder-test-stability.yaml @@ -73,7 +73,7 @@ jobs: GOMAXPROCS: ${{ runner.os == 'Windows' && 1 || 2 }} run: gotestsum --junitfile="gotests.xml" --packages="./..." -- -covermode=atomic -coverprofile="gotests.coverage" - -timeout=3m -count=$GOCOUNT -race -short -failfast + -timeout=15m -count=$GOCOUNT -race -short -failfast - name: Upload DataDog Trace if: (success() || failure()) && github.actor != 'dependabot[bot]' @@ -88,7 +88,7 @@ jobs: env: GOCOUNT: ${{ github.event.inputs.iterationCount || 15 }} run: DB=true gotestsum --junitfile="gotests.xml" --packages="./..." -- - -covermode=atomic -coverprofile="gotests.coverage" -timeout=3m + -covermode=atomic -coverprofile="gotests.coverage" -timeout=15m -count=$GOCOUNT -race -parallel=2 -failfast - name: Upload DataDog Trace From cf05b467e38c5b2a809356e77b2d124a4dc4af97 Mon Sep 17 00:00:00 2001 From: Bryan Phelps Date: Wed, 23 Feb 2022 05:59:45 +0000 Subject: [PATCH 11/15] Try default of 10 to avoid limit on simultaneously alive goroutines --- .github/workflows/coder-test-stability.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/coder-test-stability.yaml b/.github/workflows/coder-test-stability.yaml index a67d335c5dec7..333e9e58d02f6 100644 --- a/.github/workflows/coder-test-stability.yaml +++ b/.github/workflows/coder-test-stability.yaml @@ -18,7 +18,7 @@ on: iterationCount: description: 'Iteration Count' required: false - default: '15' + default: '10' # Cancel in-progress runs for pull requests when developers push # additional changes, and serialize builds in branches. @@ -69,7 +69,7 @@ jobs: - name: Test with Mock Database shell: bash env: - GOCOUNT: ${{ github.event.inputs.iterationCount || 15 }} + GOCOUNT: ${{ github.event.inputs.iterationCount || 10 }} GOMAXPROCS: ${{ runner.os == 'Windows' && 1 || 2 }} run: gotestsum --junitfile="gotests.xml" --packages="./..." -- -covermode=atomic -coverprofile="gotests.coverage" @@ -86,7 +86,7 @@ jobs: - name: Test with PostgreSQL Database if: runner.os == 'Linux' env: - GOCOUNT: ${{ github.event.inputs.iterationCount || 15 }} + GOCOUNT: ${{ github.event.inputs.iterationCount || 10 }} run: DB=true gotestsum --junitfile="gotests.xml" --packages="./..." -- -covermode=atomic -coverprofile="gotests.coverage" -timeout=15m -count=$GOCOUNT -race -parallel=2 -failfast From 636c911e3d6198962a74fcd524f2556e03b2da4c Mon Sep 17 00:00:00 2001 From: Bryan Phelps Date: Wed, 23 Feb 2022 18:39:58 +0000 Subject: [PATCH 12/15] Increase timeout for postgres tests --- .github/workflows/coder-test-stability.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coder-test-stability.yaml b/.github/workflows/coder-test-stability.yaml index 333e9e58d02f6..b7892683c301d 100644 --- a/.github/workflows/coder-test-stability.yaml +++ b/.github/workflows/coder-test-stability.yaml @@ -88,7 +88,7 @@ jobs: env: GOCOUNT: ${{ github.event.inputs.iterationCount || 10 }} run: DB=true gotestsum --junitfile="gotests.xml" --packages="./..." -- - -covermode=atomic -coverprofile="gotests.coverage" -timeout=15m + -covermode=atomic -coverprofile="gotests.coverage" -timeout=30m -count=$GOCOUNT -race -parallel=2 -failfast - name: Upload DataDog Trace From 14345dfb0f716fa2d43c8a2e66f13321483a1e36 Mon Sep 17 00:00:00 2001 From: Bryan Phelps Date: Wed, 23 Feb 2022 18:49:45 +0000 Subject: [PATCH 13/15] Remove unnecessary stability prefix, because we send up the workflow name --- .github/workflows/coder-test-stability.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coder-test-stability.yaml b/.github/workflows/coder-test-stability.yaml index b7892683c301d..f67f70fd339a9 100644 --- a/.github/workflows/coder-test-stability.yaml +++ b/.github/workflows/coder-test-stability.yaml @@ -80,7 +80,7 @@ jobs: env: DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} DD_DATABASE: fake - GIT_COMMIT_MESSAGE: stability-${{ github.event.head_commit.message }} + GIT_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: go run scripts/datadog-cireport/main.go gotests.xml - name: Test with PostgreSQL Database @@ -96,5 +96,5 @@ jobs: env: DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} DD_DATABASE: postgresql - GIT_COMMIT_MESSAGE: stability-${{ github.event.head_commit.message }} + GIT_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: go run scripts/datadog-cireport/main.go gotests.xml \ No newline at end of file From 68f6ad1eb3e3b6c10ecf8709bb6fdcf8901dbcf8 Mon Sep 17 00:00:00 2001 From: G r e y Date: Sat, 19 Mar 2022 18:55:50 -0400 Subject: [PATCH 14/15] Update .github/workflows/coder-test-stability.yaml --- .github/workflows/coder-test-stability.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coder-test-stability.yaml b/.github/workflows/coder-test-stability.yaml index f67f70fd339a9..15463a30a45db 100644 --- a/.github/workflows/coder-test-stability.yaml +++ b/.github/workflows/coder-test-stability.yaml @@ -97,4 +97,4 @@ jobs: DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} DD_DATABASE: postgresql GIT_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - run: go run scripts/datadog-cireport/main.go gotests.xml \ No newline at end of file + run: go run scripts/datadog-cireport/main.go gotests.xml From 3dd10a4969e5e8c967431570ce3a98ad96b25203 Mon Sep 17 00:00:00 2001 From: G r e y Date: Sat, 19 Mar 2022 19:00:46 -0400 Subject: [PATCH 15/15] Change branch back to main --- .github/workflows/coder-test-stability.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/coder-test-stability.yaml b/.github/workflows/coder-test-stability.yaml index 15463a30a45db..a4d055d8928e0 100644 --- a/.github/workflows/coder-test-stability.yaml +++ b/.github/workflows/coder-test-stability.yaml @@ -8,9 +8,7 @@ on: pull_request: branches: - # TODO: Remove before merge - # - main - - "*" + - main paths: - .github/workflows/coder-test-stability.yaml workflow_dispatch: