From 9c808b486328b65c61e67591b3d56cd6f731bec5 Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Wed, 30 Apr 2025 12:01:38 +0000 Subject: [PATCH 1/4] persist go build cache --- .github/actions/setup-go/action.yaml | 30 +++++++++++++++++--- .github/actions/test-cache/upload/action.yml | 6 +++- .github/workflows/ci.yaml | 30 +++++++++++++++++++- cli/gitssh_test.go | 2 ++ 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/.github/actions/setup-go/action.yaml b/.github/actions/setup-go/action.yaml index 76b7c5d87d206..a2b4906b80955 100644 --- a/.github/actions/setup-go/action.yaml +++ b/.github/actions/setup-go/action.yaml @@ -5,6 +5,13 @@ inputs: version: description: "The Go version to use." default: "1.24.2" + build-cache-path: + description: "The path to the build cache." + default: "" + required: false + cache: + description: "Whether to cache the build cache." + default: "true" runs: using: "composite" steps: @@ -12,13 +19,28 @@ runs: uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: go-version: ${{ inputs.version }} - - - name: Install gotestsum + cache: ${{ inputs.cache }} + - name: Install gotestsum and mtimehash shell: bash - run: go install gotest.tools/gotestsum@latest + run: | + export BUILD_CACHE_PATH=${{ inputs.build-cache-path }} + if [ -n "${BUILD_CACHE_PATH}" ]; then + mkdir -p "${BUILD_CACHE_PATH}" + export GOCACHE="${BUILD_CACHE_PATH}/build" + export GOMODCACHE="${BUILD_CACHE_PATH}/mod" + fi + go install gotest.tools/gotestsum@latest + go install github.com/slsyy/mtimehash/cmd/mtimehash@latest # It isn't necessary that we ever do this, but it helps # separate the "setup" from the "run" times. - name: go mod download shell: bash - run: go mod download -x + run: | + export BUILD_CACHE_PATH=${{ inputs.build-cache-path }} + if [ -n "${BUILD_CACHE_PATH}" ]; then + mkdir -p "${BUILD_CACHE_PATH}" + export GOCACHE="${BUILD_CACHE_PATH}/build" + export GOMODCACHE="${BUILD_CACHE_PATH}/mod" + fi + go mod download -x diff --git a/.github/actions/test-cache/upload/action.yml b/.github/actions/test-cache/upload/action.yml index a4d524164c74c..d60cd3e25e0df 100644 --- a/.github/actions/test-cache/upload/action.yml +++ b/.github/actions/test-cache/upload/action.yml @@ -9,11 +9,15 @@ inputs: required: true # This path is defined in testutil/cache.go default: "~/.cache/coderv2-test" + override-condition: + description: "Override condition" + required: false + default: "false" runs: using: "composite" steps: - name: Upload test cache - if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ github.ref == 'refs/heads/main' || inputs.override-condition == 'true' }} uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: ${{ inputs.cache-path }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cb1260f2ee767..b894e665dd9a8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -313,7 +313,7 @@ jobs: run: ./scripts/check_unstaged.sh test-go: - runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'depot-ubuntu-22.04-4' || matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'windows-latest-16-cores' || matrix.os }} + runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'depot-ubuntu-22.04-4' || matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'depot-windows-2022-16' || matrix.os }} needs: changes if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main' timeout-minutes: 20 @@ -335,8 +335,25 @@ jobs: with: fetch-depth: 1 + - name: Download Go Build Cache + id: download-go-build-cache + uses: ./.github/actions/test-cache/download + if: runner.os == 'Windows' + with: + key-prefix: test-go-build-11-${{ runner.os }}-${{ runner.arch }} + cache-path: "~/.cache/go-cache" + - name: Setup Go uses: ./.github/actions/setup-go + with: + build-cache-path: ~/.cache/go-cache + cache: false + + - name: Normalize File and Directory Timestamps + shell: bash + run: | + find . -type f ! -path ./.git/\*\* | mtimehash + find . -type d ! -path ./.git/\*\* -exec touch -t 200601010000 {} + - name: Setup Terraform uses: ./.github/actions/setup-tf @@ -368,6 +385,9 @@ jobs: touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile fi export TS_DEBUG_DISCO=true + export GOCACHE=~/.cache/go-cache/build + export GOMODCACHE=~/.cache/go-cache/mod + echo "PATH: $PATH" gotestsum --junitfile="gotests.xml" --jsonfile="gotests.json" \ --packages="./..." -- $PARALLEL_FLAG -short -failfast @@ -376,6 +396,14 @@ jobs: with: cache-key: ${{ steps.download-cache.outputs.cache-key }} + - name: Upload Go Build Cache + if: runner.os == 'Windows' + uses: ./.github/actions/test-cache/upload + with: + cache-key: ${{ steps.download-go-build-cache.outputs.cache-key }}-${{ github.run_id }}-${{ github.run_attempt }} + cache-path: "~/.cache/go-cache" + override-condition: "true" + - name: Upload test stats to Datadog timeout-minutes: 1 continue-on-error: true diff --git a/cli/gitssh_test.go b/cli/gitssh_test.go index 6d574ae651aec..81cc3940357c6 100644 --- a/cli/gitssh_test.go +++ b/cli/gitssh_test.go @@ -117,6 +117,8 @@ func TestGitSSH(t *testing.T) { t.Run("Dial", func(t *testing.T) { t.Parallel() + fmt.Println("Dial") + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) defer cancel() From 6db82cab75ae2338e6502ec96ddf39cebf75b2a3 Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Thu, 1 May 2025 10:09:54 +0000 Subject: [PATCH 2/4] terraform --- .github/actions/setup-tf/action.yaml | 6 ++++++ .github/workflows/ci.yaml | 8 -------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/actions/setup-tf/action.yaml b/.github/actions/setup-tf/action.yaml index a29d107826ad8..f9a270b99b77b 100644 --- a/.github/actions/setup-tf/action.yaml +++ b/.github/actions/setup-tf/action.yaml @@ -5,7 +5,13 @@ runs: using: "composite" steps: - name: Install Terraform + if: ${{ runner.os != 'Windows' }} uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2 with: terraform_version: 1.11.4 terraform_wrapper: false + - name: Install Terraform on Windows + if: ${{ runner.os == 'Windows' }} + shell: bash + run: | + choco install terraform --version=1.11.4 -y diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b894e665dd9a8..781b7a3acf650 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -404,14 +404,6 @@ jobs: cache-path: "~/.cache/go-cache" override-condition: "true" - - name: Upload test stats to Datadog - timeout-minutes: 1 - continue-on-error: true - uses: ./.github/actions/upload-datadog - if: success() || failure() - with: - api-key: ${{ secrets.DATADOG_API_KEY }} - # We don't run the full test-suite for Windows & MacOS, so we just run the CLI tests on every PR. # We run the test suite in test-go-pg, including CLI. test-cli: From 79f1e47ea7386be0086d2000c86c4ff9a9286b38 Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Thu, 1 May 2025 10:33:10 +0000 Subject: [PATCH 3/4] ramdisk --- .github/workflows/ci.yaml | 63 +++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 781b7a3acf650..dbe16c03e2b11 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -335,18 +335,23 @@ jobs: with: fetch-depth: 1 + # Sets up the ImDisk toolkit for Windows and creates a RAM disk on drive R:. + - name: Setup ImDisk + if: runner.os == 'Windows' + uses: ./.github/actions/setup-imdisk + - name: Download Go Build Cache id: download-go-build-cache uses: ./.github/actions/test-cache/download if: runner.os == 'Windows' with: key-prefix: test-go-build-11-${{ runner.os }}-${{ runner.arch }} - cache-path: "~/.cache/go-cache" + cache-path: "/r/.cache/go-cache" - name: Setup Go uses: ./.github/actions/setup-go with: - build-cache-path: ~/.cache/go-cache + build-cache-path: /r/.cache/go-cache cache: false - name: Normalize File and Directory Timestamps @@ -364,32 +369,32 @@ jobs: with: key-prefix: test-go-${{ runner.os }}-${{ runner.arch }} - - name: Test with Mock Database - id: test - shell: bash - run: | - # if macOS, install google-chrome for scaletests. As another concern, - # should we really have this kind of external dependency requirement - # on standard CI? - if [ "${{ matrix.os }}" == "macos-latest" ]; then - brew install google-chrome - fi - - # By default Go will use the number of logical CPUs, which - # is a fine default. - PARALLEL_FLAG="" - - # macOS will output "The default interactive shell is now zsh" - # intermittently in CI... - if [ "${{ matrix.os }}" == "macos-latest" ]; then - touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile - fi - export TS_DEBUG_DISCO=true - export GOCACHE=~/.cache/go-cache/build - export GOMODCACHE=~/.cache/go-cache/mod - echo "PATH: $PATH" - gotestsum --junitfile="gotests.xml" --jsonfile="gotests.json" \ - --packages="./..." -- $PARALLEL_FLAG -short -failfast + # - name: Test with Mock Database + # id: test + # shell: bash + # run: | + # # if macOS, install google-chrome for scaletests. As another concern, + # # should we really have this kind of external dependency requirement + # # on standard CI? + # if [ "${{ matrix.os }}" == "macos-latest" ]; then + # brew install google-chrome + # fi + + # # By default Go will use the number of logical CPUs, which + # # is a fine default. + # PARALLEL_FLAG="" + + # # macOS will output "The default interactive shell is now zsh" + # # intermittently in CI... + # if [ "${{ matrix.os }}" == "macos-latest" ]; then + # touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile + # fi + # export TS_DEBUG_DISCO=true + # export GOCACHE=/r/.cache/go-cache/build + # export GOMODCACHE=/r/.cache/go-cache/mod + # echo "PATH: $PATH" + # gotestsum --junitfile="gotests.xml" --jsonfile="gotests.json" \ + # --packages="./..." -- $PARALLEL_FLAG -short -failfast - name: Upload Test Cache uses: ./.github/actions/test-cache/upload @@ -401,7 +406,7 @@ jobs: uses: ./.github/actions/test-cache/upload with: cache-key: ${{ steps.download-go-build-cache.outputs.cache-key }}-${{ github.run_id }}-${{ github.run_attempt }} - cache-path: "~/.cache/go-cache" + cache-path: "/r/.cache/go-cache" override-condition: "true" # We don't run the full test-suite for Windows & MacOS, so we just run the CLI tests on every PR. From 198112f3ab0550b88112f16b7e020cbc4607fd15 Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Thu, 1 May 2025 18:27:49 +0000 Subject: [PATCH 4/4] wush --- .github/actions/setup-go/action.yaml | 46 ++++----- .github/actions/setup-imdisk/action.yaml | 25 ++++- .github/workflows/ci.yaml | 117 +++-------------------- 3 files changed, 59 insertions(+), 129 deletions(-) diff --git a/.github/actions/setup-go/action.yaml b/.github/actions/setup-go/action.yaml index a2b4906b80955..d8c8b14db67b1 100644 --- a/.github/actions/setup-go/action.yaml +++ b/.github/actions/setup-go/action.yaml @@ -20,27 +20,27 @@ runs: with: go-version: ${{ inputs.version }} cache: ${{ inputs.cache }} - - name: Install gotestsum and mtimehash - shell: bash - run: | - export BUILD_CACHE_PATH=${{ inputs.build-cache-path }} - if [ -n "${BUILD_CACHE_PATH}" ]; then - mkdir -p "${BUILD_CACHE_PATH}" - export GOCACHE="${BUILD_CACHE_PATH}/build" - export GOMODCACHE="${BUILD_CACHE_PATH}/mod" - fi - go install gotest.tools/gotestsum@latest - go install github.com/slsyy/mtimehash/cmd/mtimehash@latest + # - name: Install gotestsum and mtimehash + # shell: bash + # run: | + # export BUILD_CACHE_PATH=${{ inputs.build-cache-path }} + # if [ -n "${BUILD_CACHE_PATH}" ]; then + # mkdir -p "${BUILD_CACHE_PATH}" + # export GOCACHE="${BUILD_CACHE_PATH}/build" + # export GOMODCACHE="${BUILD_CACHE_PATH}/mod" + # fi + # go install gotest.tools/gotestsum@latest + # go install github.com/slsyy/mtimehash/cmd/mtimehash@latest - # It isn't necessary that we ever do this, but it helps - # separate the "setup" from the "run" times. - - name: go mod download - shell: bash - run: | - export BUILD_CACHE_PATH=${{ inputs.build-cache-path }} - if [ -n "${BUILD_CACHE_PATH}" ]; then - mkdir -p "${BUILD_CACHE_PATH}" - export GOCACHE="${BUILD_CACHE_PATH}/build" - export GOMODCACHE="${BUILD_CACHE_PATH}/mod" - fi - go mod download -x + # # It isn't necessary that we ever do this, but it helps + # # separate the "setup" from the "run" times. + # - name: go mod download + # shell: bash + # run: | + # export BUILD_CACHE_PATH=${{ inputs.build-cache-path }} + # if [ -n "${BUILD_CACHE_PATH}" ]; then + # mkdir -p "${BUILD_CACHE_PATH}" + # export GOCACHE="${BUILD_CACHE_PATH}/build" + # export GOMODCACHE="${BUILD_CACHE_PATH}/mod" + # fi + # go mod download -x diff --git a/.github/actions/setup-imdisk/action.yaml b/.github/actions/setup-imdisk/action.yaml index 52ef7eb08fd81..b2d81ca3d691a 100644 --- a/.github/actions/setup-imdisk/action.yaml +++ b/.github/actions/setup-imdisk/action.yaml @@ -1,7 +1,7 @@ name: "Setup ImDisk" if: runner.os == 'Windows' description: | - Sets up the ImDisk toolkit for Windows and creates a RAM disk on drive R:. + Sets up the ImDisk toolkit for Windows and backs often-used paths with RAM disks runs: using: "composite" steps: @@ -22,6 +22,27 @@ runs: install.bat /silent - name: Create RAM Disk - shell: cmd + shell: bash run: | imdisk -a -s 4096M -m R: -p "/fs:ntfs /q /y" + + # bash + mkdir "$RUNNER_TEMP"_tmp + cp -r "$RUNNER_TEMP"/. "$RUNNER_TEMP"_tmp + rm -rf "$RUNNER_TEMP"/* + imdisk -a -s 8192M -m "$RUNNER_TEMP" -p "/fs:ntfs /q /y" + cp -r "$RUNNER_TEMP"_tmp/. "$RUNNER_TEMP" + rm -rf "$RUNNER_TEMP"_tmp + + cd "$RUNNER_TEMP" + export GOCACHE="$(go env GOCACHE)" + export GOMODCACHE="$(go env GOMODCACHE)" + rm -rf "$GOCACHE" + rm -rf "$GOMODCACHE" + mkdir -p "$GOCACHE" + mkdir -p "$GOMODCACHE" + imdisk -a -s 8192M -m "$GOCACHE" -p "/fs:ntfs /q /y" + imdisk -a -s 8192M -m "$GOMODCACHE" -p "/fs:ntfs /q /y" + + rm -rf "$GITHUB_WORKSPACE"/* + imdisk -a -s 2048M -m "$GITHUB_WORKSPACE" -p "/fs:ntfs /q /y" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index dbe16c03e2b11..8e4a1c389c845 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -236,45 +236,6 @@ jobs: with: fetch-depth: 1 - - name: Setup Node - uses: ./.github/actions/setup-node - - - name: Setup Go - uses: ./.github/actions/setup-go - - - name: Setup sqlc - uses: ./.github/actions/setup-sqlc - - - name: Setup Terraform - uses: ./.github/actions/setup-tf - - - name: go install tools - uses: ./.github/actions/setup-go-tools - - - name: Install Protoc - run: | - mkdir -p /tmp/proto - pushd /tmp/proto - curl -L -o protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v23.4/protoc-23.4-linux-x86_64.zip - unzip protoc.zip - cp -r ./bin/* /usr/local/bin - cp -r ./include /usr/local/bin/include - popd - - - name: make gen - run: | - # Remove golden files to detect discrepancy in generated files. - make clean/golden-files - # Notifications require DB, we could start a DB instance here but - # let's just restore for now. - git checkout -- coderd/notifications/testdata/rendered-templates - # no `-j` flag as `make` fails with: - # coderd/rbac/object_gen.go:1:1: syntax error: package statement must be first - make --output-sync -B gen - - - name: Check for unstaged files - run: ./scripts/check_unstaged.sh - fmt: needs: changes if: needs.changes.outputs.offlinedocs-only == 'false' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main' @@ -316,7 +277,7 @@ jobs: runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'depot-ubuntu-22.04-4' || matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'depot-windows-2022-16' || matrix.os }} needs: changes if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main' - timeout-minutes: 20 + timeout-minutes: 90 strategy: fail-fast: false matrix: @@ -326,88 +287,36 @@ jobs: - windows-2022 steps: - name: Harden Runner + if: runner.os == 'Linux' uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0 with: egress-policy: audit - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - # Sets up the ImDisk toolkit for Windows and creates a RAM disk on drive R:. - name: Setup ImDisk if: runner.os == 'Windows' - uses: ./.github/actions/setup-imdisk + uses: hugodutka/setup-imdisk@e6437cdfa0a7e110bc3f560da8c04b3c3fcc4135 - - name: Download Go Build Cache - id: download-go-build-cache - uses: ./.github/actions/test-cache/download - if: runner.os == 'Windows' + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - key-prefix: test-go-build-11-${{ runner.os }}-${{ runner.arch }} - cache-path: "/r/.cache/go-cache" + fetch-depth: 1 - name: Setup Go uses: ./.github/actions/setup-go with: - build-cache-path: /r/.cache/go-cache + version: "" cache: false - - name: Normalize File and Directory Timestamps + - name: Wush + if: always() && runner.os == 'Windows' shell: bash run: | - find . -type f ! -path ./.git/\*\* | mtimehash - find . -type d ! -path ./.git/\*\* -exec touch -t 200601010000 {} + - - - name: Setup Terraform - uses: ./.github/actions/setup-tf - - - name: Download Test Cache - id: download-cache - uses: ./.github/actions/test-cache/download - with: - key-prefix: test-go-${{ runner.os }}-${{ runner.arch }} - - # - name: Test with Mock Database - # id: test - # shell: bash - # run: | - # # if macOS, install google-chrome for scaletests. As another concern, - # # should we really have this kind of external dependency requirement - # # on standard CI? - # if [ "${{ matrix.os }}" == "macos-latest" ]; then - # brew install google-chrome - # fi - - # # By default Go will use the number of logical CPUs, which - # # is a fine default. - # PARALLEL_FLAG="" - - # # macOS will output "The default interactive shell is now zsh" - # # intermittently in CI... - # if [ "${{ matrix.os }}" == "macos-latest" ]; then - # touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile - # fi - # export TS_DEBUG_DISCO=true - # export GOCACHE=/r/.cache/go-cache/build - # export GOMODCACHE=/r/.cache/go-cache/mod - # echo "PATH: $PATH" - # gotestsum --junitfile="gotests.xml" --jsonfile="gotests.json" \ - # --packages="./..." -- $PARALLEL_FLAG -short -failfast - - - name: Upload Test Cache - uses: ./.github/actions/test-cache/upload - with: - cache-key: ${{ steps.download-cache.outputs.cache-key }} + curl -L -o wush.zip https://github.com/coder/wush/releases/download/v0.4.1/wush_0.4.1_windows_amd64.zip + unzip -o wush.zip - - name: Upload Go Build Cache - if: runner.os == 'Windows' - uses: ./.github/actions/test-cache/upload - with: - cache-key: ${{ steps.download-go-build-cache.outputs.cache-key }}-${{ github.run_id }}-${{ github.run_attempt }} - cache-path: "/r/.cache/go-cache" - override-condition: "true" + chmod +x wush.exe + ./wush.exe serve # We don't run the full test-suite for Windows & MacOS, so we just run the CLI tests on every PR. # We run the test suite in test-go-pg, including CLI.