Skip to content

Commit 1d00ac8

Browse files
hugodutkahugodutka
committed
enable go test cache in test-go-pg
1 parent 5e2fd9d commit 1d00ac8

File tree

5 files changed

+180
-41
lines changed

5 files changed

+180
-41
lines changed

‎.github/actions/setup-go-paths/action.yml

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
name: "Setup Go Paths"
2+
description: Overrides Go paths like GOCACHE and GOMODCACHE to use temporary directories.
3+
outputs:
4+
gocache:
5+
description: "Value of GOCACHE"
6+
value: ${{ steps.paths.outputs.gocache }}
7+
gomodcache:
8+
description: "Value of GOMODCACHE"
9+
value: ${{ steps.paths.outputs.gomodcache }}
10+
gopath:
11+
description: "Value of GOPATH"
12+
value: ${{ steps.paths.outputs.gopath }}
13+
gotmp:
14+
description: "Value of GOTMPDIR"
15+
value: ${{ steps.paths.outputs.gotmp }}
16+
cached-dirs:
17+
description: "Go directories that should be cached between CI runs"
18+
value: ${{ steps.paths.outputs.cached-dirs }}
19+
runs:
20+
using: "composite"
21+
steps:
22+
- name: Override Go paths
23+
id: paths
24+
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
25+
with:
26+
script: |
27+
const path = require('path');
28+
29+
// RUNNER_TEMP should be backed by a RAM disk on Windows if
30+
// coder/setup-ramdisk-action was used
31+
const runnerTemp = process.env.RUNNER_TEMP;
32+
const gocacheDir = path.join(runnerTemp, 'go-cache');
33+
const gomodcacheDir = path.join(runnerTemp, 'go-mod-cache');
34+
const gopathDir = path.join(runnerTemp, 'go-path');
35+
const gotmpDir = path.join(runnerTemp, 'go-tmp');
36+
37+
core.exportVariable('GOCACHE', gocacheDir);
38+
core.exportVariable('GOMODCACHE', gomodcacheDir);
39+
core.exportVariable('GOPATH', gopathDir);
40+
core.exportVariable('GOTMPDIR', gotmpDir);
41+
42+
core.setOutput('gocache', gocacheDir);
43+
core.setOutput('gomodcache', gomodcacheDir);
44+
core.setOutput('gopath', gopathDir);
45+
core.setOutput('gotmp', gotmpDir);
46+
47+
const cachedDirs = `${gocacheDir}\n${gomodcacheDir}`;
48+
core.setOutput('cached-dirs', cachedDirs);
49+
50+
- name: Create directories
51+
shell: bash
52+
run: |
53+
set -e
54+
mkdir -p "$GOCACHE"
55+
mkdir -p "$GOMODCACHE"
56+
mkdir -p "$GOPATH"
57+
mkdir -p "$GOTMPDIR"

‎.github/actions/setup-go/action.yaml

Lines changed: 8 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -8,42 +8,26 @@ inputs:
88
use-preinstalled-go:
99
description: "Whether to use preinstalled Go."
1010
default: "false"
11-
use-temp-cache-dirs:
12-
description: "Whether to use temporary GOCACHE and GOMODCACHE directories."
13-
default: "false"
11+
use-cache:
12+
description: "Whether to use the cache."
13+
default: "true"
1414
runs:
1515
using: "composite"
1616
steps:
17-
- name: Override GOCACHE and GOMODCACHE
18-
shell: bash
19-
if: inputs.use-temp-cache-dirs == 'true'
20-
run: |
21-
# cd to another directory to ensure we're not inside a Go project.
22-
# That'd trigger Go to download the toolchain for that project.
23-
cd "$RUNNER_TEMP"
24-
# RUNNER_TEMP should be backed by a RAM disk on Windows if
25-
# coder/setup-ramdisk-action was used
26-
export GOCACHE_DIR="$RUNNER_TEMP""\go-cache"
27-
export GOMODCACHE_DIR="$RUNNER_TEMP""\go-mod-cache"
28-
export GOPATH_DIR="$RUNNER_TEMP""\go-path"
29-
export GOTMP_DIR="$RUNNER_TEMP""\go-tmp"
30-
mkdir -p "$GOCACHE_DIR"
31-
mkdir -p "$GOMODCACHE_DIR"
32-
mkdir -p "$GOPATH_DIR"
33-
mkdir -p "$GOTMP_DIR"
34-
go env -w GOCACHE="$GOCACHE_DIR"
35-
go env -w GOMODCACHE="$GOMODCACHE_DIR"
36-
go env -w GOPATH="$GOPATH_DIR"
37-
go env -w GOTMPDIR="$GOTMP_DIR"
3817
- name: Setup Go
3918
uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
4019
with:
4120
go-version: ${{ inputs.use-preinstalled-go == 'false' && inputs.version || '' }}
21+
cache: ${{ inputs.use-cache }}
4222

4323
- name: Install gotestsum
4424
shell: bash
4525
run: go install gotest.tools/gotestsum@0d9599e513d70e5792bb9334869f82f6e8b53d4d # main as of 2025-05-15
4626

27+
- name: Install mtimehash
28+
shell: bash
29+
run: go install github.com/slsyy/mtimehash/cmd/mtimehash@a6b5da4ed2c4a40e7b805534b004e9fde7b53ce0 # v1.0.0
30+
4731
# It isn't necessary that we ever do this, but it helps
4832
# separate the "setup" from the "run" times.
4933
- name: go mod download

‎.github/workflows/ci.yaml

Lines changed: 57 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -336,21 +336,23 @@ jobs:
336336
# a separate repository to allow its use before actions/checkout.
337337
- name: Setup RAM Disks
338338
if: runner.os == 'Windows'
339-
uses: coder/setup-ramdisk-action@81c5c441bda00c6c3d6bcee2e5a33ed4aadbbcc1
339+
uses: coder/setup-ramdisk-action@a4b59caa8be2e88c348abeef042d7c1a33d8743e
340340

341341
- name: Checkout
342342
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
343343
with:
344344
fetch-depth: 1
345345

346+
- name: Setup Go Paths
347+
uses: ./.github/actions/setup-go-paths
348+
346349
- name: Setup Go
347350
uses: ./.github/actions/setup-go
348351
with:
349352
# Runners have Go baked-in and Go will automatically
350353
# download the toolchain configured in go.mod, so we don't
351354
# need to reinstall it. It's faster on Windows runners.
352355
use-preinstalled-go: ${{ runner.os == 'Windows' }}
353-
use-temp-cache-dirs: ${{ runner.os == 'Windows' }}
354356

355357
- name: Setup Terraform
356358
uses: ./.github/actions/setup-tf
@@ -490,21 +492,33 @@ jobs:
490492
# a separate repository to allow its use before actions/checkout.
491493
- name: Setup RAM Disks
492494
if: runner.os == 'Windows'
493-
uses: coder/setup-ramdisk-action@79dacfe70c47ad6d6c0dd7f45412368802641439
495+
uses: coder/setup-ramdisk-action@a4b59caa8be2e88c348abeef042d7c1a33d8743e
494496

495497
- name: Checkout
496498
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
497499
with:
498500
fetch-depth: 1
499501

502+
- name: Setup Go Paths
503+
id: go-paths
504+
uses: ./.github/actions/setup-go-paths
505+
506+
- name: Download Go Build Cache
507+
id: download-go-build-cache
508+
uses: ./.github/actions/test-cache/download
509+
with:
510+
key-prefix: test-go-build-${{ runner.os }}-${{ runner.arch }}
511+
cache-path: ${{ steps.go-paths.outputs.cached-dirs }}
512+
500513
- name: Setup Go
501514
uses: ./.github/actions/setup-go
502515
with:
503516
# Runners have Go baked-in and Go will automatically
504517
# download the toolchain configured in go.mod, so we don't
505518
# need to reinstall it. It's faster on Windows runners.
506519
use-preinstalled-go: ${{ runner.os == 'Windows' }}
507-
use-temp-cache-dirs: ${{ runner.os == 'Windows' }}
520+
# Cache is already downloaded above
521+
use-cache: false
508522

509523
- name: Setup Terraform
510524
uses: ./.github/actions/setup-tf
@@ -515,14 +529,27 @@ jobs:
515529
with:
516530
key-prefix: test-go-pg-${{ runner.os }}-${{ runner.arch }}
517531

532+
- name: Normalize File and Directory Timestamps
533+
shell: bash
534+
# Normalize file modification timestamps so that go test can use the
535+
# cache from the previous CI run. See https://github.com/golang/go/issues/58571
536+
# for more details.
537+
run: |
538+
find . -type f ! -path ./.git/\*\* | mtimehash
539+
find . -type d ! -path ./.git/\*\* -exec touch -t 200601010000 {} +
540+
518541
- name: Test with PostgreSQL Database
519542
env:
520543
POSTGRES_VERSION: "13"
521544
TS_DEBUG_DISCO: "true"
522545
LC_CTYPE: "en_US.UTF-8"
523546
LC_ALL: "en_US.UTF-8"
547+
524548
shell: bash
525549
run: |
550+
set -o errexit
551+
set -o pipefail
552+
526553
if [ "${{ runner.os }}" == "Windows" ]; then
527554
# Create a temp dir on the R: ramdisk drive for Windows. The default
528555
# C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755
@@ -533,6 +560,8 @@ jobs:
533560
mkdir -p /tmp/tmpfs
534561
sudo mount_tmpfs -o noowners -s 8g /tmp/tmpfs
535562
go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg
563+
elif [ "${{ runner.os }}" == "Linux" ]; then
564+
make test-postgres-docker
536565
fi
537566
538567
# if macOS, install google-chrome for scaletests
@@ -542,10 +571,6 @@ jobs:
542571
brew install google-chrome
543572
fi
544573
545-
# By default Go will use the number of logical CPUs, which
546-
# is a fine default.
547-
PARALLEL_FLAG=""
548-
549574
# macOS will output "The default interactive shell is now zsh"
550575
# intermittently in CI...
551576
if [ "${{ matrix.os }}" == "macos-latest" ]; then
@@ -572,16 +597,32 @@ jobs:
572597
NUM_PARALLEL_TESTS=8
573598
fi
574599
575-
if [ "${{ runner.os }}" == "Linux" ]; then
576-
make test-postgres
577-
else
578-
# We rerun failing tests to counteract flakiness coming from Postgres
579-
# choking on macOS and Windows sometimes.
580-
DB=ci gotestsum --rerun-fails=2 --rerun-fails-max-failures=50 \
581-
--format standard-quiet --packages "./..." \
582-
-- -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS -count=1
600+
# by default, run tests with cache
601+
TESTCOUNT=""
602+
if [ "${{ github.ref }}" == "refs/heads/main" ]; then
603+
# on main, run tests without cache
604+
TESTCOUNT="-count=1"
583605
fi
584606
607+
mkdir -p "$RUNNER_TEMP/sym"
608+
source scripts/normalize_path.sh
609+
# terraform gets installed in a random directory, so we need to normalize
610+
# the path to the terraform binary or a bunch of cached tests will be
611+
# invalidated. See scripts/normalize_path.sh for more details.
612+
normalize_path_with_symlinks "$RUNNER_TEMP/sym" "$(dirname $(which terraform))"
613+
614+
# We rerun failing tests to counteract flakiness coming from Postgres
615+
# choking on macOS and Windows sometimes.
616+
DB=ci gotestsum --rerun-fails=2 --rerun-fails-max-failures=50 \
617+
--format standard-quiet --packages "./..." \
618+
-- -timeout=20m -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS $TESTCOUNT
619+
620+
- name: Upload Go Build Cache
621+
uses: ./.github/actions/test-cache/upload
622+
with:
623+
cache-key: ${{ steps.download-go-build-cache.outputs.cache-key }}
624+
cache-path: ${{ steps.go-paths.outputs.cached-dirs }}
625+
585626
- name: Upload Test Cache
586627
uses: ./.github/actions/test-cache/upload
587628
with:

‎.github/workflows/nightly-gauntlet.yaml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,14 +53,16 @@ jobs:
5353
with:
5454
fetch-depth: 1
5555

56+
- name: Setup Go Paths
57+
uses: ./.github/actions/setup-go-paths
58+
5659
- name: Setup Go
5760
uses: ./.github/actions/setup-go
5861
with:
5962
# Runners have Go baked-in and Go will automatically
6063
# download the toolchain configured in go.mod, so we don't
6164
# need to reinstall it. It's faster on Windows runners.
6265
use-preinstalled-go: ${{ runner.os == 'Windows' }}
63-
use-temp-cache-dirs: ${{ runner.os == 'Windows' }}
6466

6567
- name: Setup Terraform
6668
uses: ./.github/actions/setup-tf

‎scripts/normalize_path.sh

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/bin/bash
2+
3+
# Call: normalize_path_with_symlinks [target_dir] [dir_prefix]
4+
#
5+
# Normalizes the PATH environment variable by replacing each directory that
6+
# begins with dir_prefix with a symbolic link in target_dir. For example, if
7+
# PATH is "/usr/bin:/bin", target_dir is /tmp, and dir_prefix is /usr, then
8+
# PATH will become "/tmp/0:/bin", where /tmp/0 links to /usr/bin.
9+
#
10+
# This is useful for ensuring that PATH is consistent across CI runs and helps
11+
# with reusing the same cache across them. Many of our go tests read the PATH
12+
# variable, and if it changes between runs, the cache gets invalidated.
13+
normalize_path_with_symlinks() {
14+
local target_dir="${1:-}"
15+
local dir_prefix="${2:-}"
16+
17+
if [[ -z "$target_dir" || -z "$dir_prefix" ]]; then
18+
echo "Usage: normalize_path_with_symlinks <target_dir> <dir_prefix>"
19+
return 1
20+
fi
21+
22+
local old_path="$PATH"
23+
local -a new_parts=()
24+
local i=0
25+
26+
IFS=':' read -ra _parts <<<"$old_path"
27+
for dir in "${_parts[@]}"; do
28+
# Skip empty components that can arise from "::"
29+
[[ -z $dir ]] && continue
30+
31+
# Skip directories that don't start with $dir_prefix
32+
if [[ "$dir" != "$dir_prefix"* ]]; then
33+
new_parts+=("$dir")
34+
continue
35+
fi
36+
37+
local link="$target_dir/$i"
38+
39+
# Replace any pre-existing file or link at $target_dir/$i
40+
if [[ -e $link || -L $link ]]; then
41+
rm -rf -- "$link"
42+
fi
43+
44+
# without MSYS ln will deepcopy the directory on Windows
45+
MSYS=winsymlinks:nativestrict ln -s -- "$dir" "$link"
46+
new_parts+=("$link")
47+
i=$((i + 1))
48+
done
49+
50+
export PATH
51+
PATH="$(
52+
IFS=':'
53+
echo "${new_parts[*]}"
54+
)"
55+
}

0 commit comments

Comments
 (0)