Skip to content

Commit fd0882b

Browse files
eranhderanhd
authored
MOD-8197 Unified CI workflows (#1304)
- Updated and added new GitHub Actions workflows for build, test, and deployment - Introduced event-based workflows for specific triggers: - `event-ci.yml`: Runs CI checks on pull requests and pushes to specific branches - `event-nightly.yml`: Executes nightly builds and tests - `event-tag.yml`: Triggers workflows on new tags for release processes - Differentiated flow-based workflows for various environments: - `flow-alpine.yml`: Handles builds and tests for Alpine Linux - `flow-linux-x86.yml`: Manages workflows for x86 Linux distributions - `flow-macos.yml`: Manages workflows for macOS environments - `flow-ubuntu-arm.yml`: Handles builds and tests for Ubuntu on ARM architecture - Removed obsolete workflows - Added new installation scripts for various Linux distributions and macOS - Modified existing scripts and configurations for better compatibility and performance
1 parent 93d8c18 commit fd0882b

33 files changed

+905
-1332
lines changed

‎.circleci/config.yml

-553
This file was deleted.

‎.github/actions/build-json-module-and-redis-with-cargo/action.yml

+2-1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ runs:
1919
- name: Build module
2020
shell: bash
2121
run: |
22-
. "$HOME/.cargo/env"
22+
. $HOME/.cargo/env
23+
echo "source $HOME/.cargo/env" >> $HOME/.bash_profile
2324
cargo build --release
2425
cp $(realpath ./target/release)/librejson.so $(realpath ./target/release)/rejson.so

‎.github/actions/build-json-module-and-redis/action.yml

-4
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,6 @@ description: |
33
Build JSON module and Redis Server
44
55
inputs:
6-
redis-ref:
7-
description: 'Redis version to build'
8-
required: true
9-
default: '7.4'
106
sanitizer:
117
type: string
128

‎.github/actions/make-pack/action.yml

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
name: Run make pack module script
2+
3+
runs:
4+
using: composite
5+
steps:
6+
- name: Pack module
7+
shell: bash
8+
run: |
9+
make pack BRANCH=$TAG_OR_BRANCH

‎.github/actions/pack-module/action.yml

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
name: Run pack module script
2+
3+
runs:
4+
using: composite
5+
steps:
6+
- name: Pack module
7+
shell: bash
8+
run: |
9+
if command -v scl_source &> /dev/null
10+
then
11+
. scl_source enable devtoolset-11 || true
12+
fi
13+
. venv/bin/activate
14+
git config --global --add safe.directory $GITHUB_WORKSPACE
15+
export PATH="$GITHUB_WORKSPACE/redis/src:$PATH"
16+
MODULE=$(realpath ./target/release/rejson.so) BRANCH=$TAG_OR_BRANCH \
17+
SHOW=1 OSNICK=${{ matrix.docker.nick }} ./sbin/pack.sh

‎.github/actions/run-tests/action.yml

+8-101
Original file line numberDiff line numberDiff line change
@@ -1,122 +1,29 @@
1-
name: Common Flow for Tests
2-
3-
# Documentation: https://redislabs.atlassian.net/wiki/spaces/DX/pages/3967844669/RediSearch+CI+refactor
1+
name: Run module tests
42

53
inputs:
6-
env:
7-
default: "ubuntu-22.04"
8-
type: string
9-
container:
10-
type: string
11-
sanitizer:
12-
type: string
13-
test-config:
14-
description: 'Test configuration environment variable (e.g. "CONFIG=tls" or "QUICK=1")'
15-
required: true
4+
run_valgrind:
5+
description: 'Run valgrind on the tests'
166
type: string
7+
default: 0
178

189
runs:
1910
using: composite
2011
steps:
21-
- name: Get Installation Mode
22-
shell: bash
23-
id: mode
24-
run: |
25-
[[ -z "${{ inputs.container }}" ]] && echo "mode=sudo" >> $GITHUB_OUTPUT || echo "mode=" >> $GITHUB_OUTPUT
26-
- name: Check if node20 is Supported
27-
id: node20
28-
uses: ./.github/actions/node20-supported
29-
with:
30-
container: ${{ inputs.container }}
31-
- name: Install git
32-
shell: bash
33-
run: |
34-
# TODO: must be changed to run a script based on the input env
35-
echo ::group::Install git
36-
${{ steps.mode.outputs.mode }} apt-get update && apt-get install -y git
37-
echo ::endgroup::
38-
- name: Setup specific
39-
shell: bash
40-
working-directory: .install
41-
run: |
42-
echo ::group::OS-Specific Setup
43-
./install_script.sh ${{ steps.mode.outputs.mode }}
44-
echo ::endgroup::
45-
echo ::group::Get Rust
46-
./getrust.sh ${{ steps.mode.outputs.mode }}
47-
echo ::endgroup::
48-
49-
- name: Full checkout (node20 supported)
50-
if: steps.node20.outputs.supported == 'true'
51-
uses: actions/checkout@v4
52-
with:
53-
submodules: recursive
54-
- name: Full checkout (node20 unsupported)
55-
if: steps.node20.outputs.supported == 'false'
56-
uses: actions/checkout@v3
57-
with:
58-
submodules: recursive
59-
60-
- name: Get Redis
61-
uses: actions/checkout@v3
62-
with:
63-
repository: redis/redis
64-
ref: 'unstable' # todo change per version/tag
65-
path: redis
66-
submodules: 'recursive'
67-
- name: Build
68-
uses: ./.github/actions/build-json-module-and-redis
69-
with:
70-
sanitizer: ${{ inputs.san }}
71-
72-
- name: Set Artifact Names
73-
shell: bash
74-
# Artifact names have to be unique, so we base them on the environment.
75-
# We also remove invalid characters from the name.
76-
id: artifact-names
77-
run: |
78-
# Invalid characters include: Double quote ", Colon :, Less than <, Greater than >, Vertical bar |, Asterisk *, Question mark ?
79-
echo "name=$(echo "${{ inputs.container || inputs.env }} ${{ runner.arch }}, Redis ${{ inputs.get-redis || 'unstable' }}" | \
80-
sed -e 's/[":\/\\<>\|*?]/_/g' -e 's/__*/_/g' -e 's/^_//' -e 's/_$//')" >> $GITHUB_OUTPUT
81-
8212
- name: Run tests
8313
shell: bash
84-
id: test
8514
run: |
8615
echo ::group::Activate virtual environment
8716
python3 -m venv venv
8817
echo "source $PWD/venv/bin/activate" >> ~/.bash_profile
89-
source venv/bin/activate
18+
. venv/bin/activate
9019
echo ::endgroup::
9120
echo ::group::Install python dependencies
9221
./.install/common_installations.sh
9322
echo ::endgroup::
9423
echo ::group::Unit tests
95-
. "$HOME/.cargo/env"
96-
make cargo_test LOG=1 CLEAR_LOGS=0 SAN=${{ inputs.san }}
24+
. $HOME/.cargo/env
25+
cargo test
9726
echo ::endgroup::
9827
echo ::group::Flow tests
99-
make pytest LOG=1 CLEAR_LOGS=0 SAN=${{ inputs.san }} ${{ inputs.test-config }}
28+
MODULE=$(realpath ./target/release/rejson.so) RLTEST_ARGS='--no-progress' ./tests/pytest/tests.sh VG=${{inputs.run_valgrind}}
10029
echo ::endgroup::
101-
env:
102-
PIP_BREAK_SYSTEM_PACKAGES: 1
103-
104-
- name: Upload test artifacts (node20 supported)
105-
if: steps.node20.outputs.supported == 'true' && steps.test.outcome == 'failure'
106-
uses: actions/upload-artifact@v4
107-
with:
108-
name: Test logs ${{ steps.artifact-names.outputs.name }}
109-
path: tests/**/logs/*.log*
110-
if-no-files-found: ignore
111-
- name: Upload test artifacts (node20 unsupported)
112-
if: steps.node20.outputs.supported == 'false' && steps.test.outcome == 'failure'
113-
uses: actions/upload-artifact@v3
114-
with:
115-
name: Test logs ${{ steps.artifact-names.outputs.name }}
116-
path: tests/**/logs/*.log*
117-
if-no-files-found: ignore
118-
119-
- name: Fail flow if tests failed
120-
shell: bash
121-
if: steps.test.outcome == 'failure'
122-
run: exit 1

‎.github/actions/san-run-tests/action.yml

+126
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
name: Common Flow for Tests
2+
3+
# Documentation: https://redislabs.atlassian.net/wiki/spaces/DX/pages/3967844669/RediSearch+CI+refactor
4+
5+
inputs:
6+
env:
7+
default: "ubuntu-22.04"
8+
type: string
9+
container:
10+
type: string
11+
sanitizer:
12+
type: string
13+
test-config:
14+
description: 'Test configuration environment variable (e.g. "CONFIG=tls" or "QUICK=1")'
15+
required: true
16+
type: string
17+
redis-ref:
18+
description: 'Redis ref to checkout'
19+
type: string
20+
required: true
21+
22+
runs:
23+
using: composite
24+
steps:
25+
- name: Get Installation Mode
26+
shell: bash
27+
id: mode
28+
run: |
29+
[[ -z "${{ inputs.container }}" ]] && echo "mode=sudo" >> $GITHUB_OUTPUT || echo "mode=" >> $GITHUB_OUTPUT
30+
- name: Check if node20 is Supported
31+
id: node20
32+
uses: ./.github/actions/node20-supported
33+
with:
34+
container: ${{ inputs.container }}
35+
- name: Install git
36+
shell: bash
37+
run: |
38+
# TODO: must be changed to run a script based on the input env
39+
echo ::group::Install git
40+
${{ steps.mode.outputs.mode }} apt-get update && apt-get install -y git
41+
echo ::endgroup::
42+
- name: Setup specific
43+
shell: bash
44+
working-directory: .install
45+
run: |
46+
echo ::group::OS-Specific Setup
47+
./install_script.sh ${{ steps.mode.outputs.mode }}
48+
echo ::endgroup::
49+
echo ::group::Get Rust
50+
./getrust.sh ${{ steps.mode.outputs.mode }}
51+
echo ::endgroup::
52+
53+
- name: Full checkout (node20 supported)
54+
if: steps.node20.outputs.supported == 'true'
55+
uses: actions/checkout@v4
56+
with:
57+
submodules: recursive
58+
- name: Full checkout (node20 unsupported)
59+
if: steps.node20.outputs.supported == 'false'
60+
uses: actions/checkout@v4
61+
with:
62+
submodules: recursive
63+
64+
- name: Get Redis
65+
uses: actions/checkout@v4
66+
with:
67+
repository: redis/redis
68+
ref: ${{ inputs.redis-ref }}
69+
path: redis
70+
submodules: 'recursive'
71+
- name: Build
72+
uses: ./.github/actions/build-json-module-and-redis
73+
with:
74+
sanitizer: ${{ inputs.san }}
75+
76+
- name: Set Artifact Names
77+
shell: bash
78+
# Artifact names have to be unique, so we base them on the environment.
79+
# We also remove invalid characters from the name.
80+
id: artifact-names
81+
run: |
82+
# Invalid characters include: Double quote ", Colon :, Less than <, Greater than >, Vertical bar |, Asterisk *, Question mark ?
83+
echo "name=$(echo "${{ inputs.container || inputs.env }} ${{ runner.arch }}, Redis ${{ inputs.redis-ref }}" | \
84+
sed -e 's/[":\/\\<>\|*?]/_/g' -e 's/__*/_/g' -e 's/^_//' -e 's/_$//')" >> $GITHUB_OUTPUT
85+
86+
- name: Run tests
87+
shell: bash
88+
id: test
89+
run: |
90+
echo ::group::Activate virtual environment
91+
python3 -m venv venv
92+
echo "source $PWD/venv/bin/activate" >> ~/.bash_profile
93+
source venv/bin/activate
94+
echo ::endgroup::
95+
echo ::group::Install python dependencies
96+
./.install/common_installations.sh
97+
echo ::endgroup::
98+
echo ::group::Unit tests
99+
. "$HOME/.cargo/env"
100+
make cargo_test LOG=1 CLEAR_LOGS=0 SAN=${{ inputs.san }}
101+
echo ::endgroup::
102+
echo ::group::Flow tests
103+
make pytest LOG=1 CLEAR_LOGS=0 SAN=${{ inputs.san }} ${{ inputs.test-config }}
104+
echo ::endgroup::
105+
env:
106+
PIP_BREAK_SYSTEM_PACKAGES: 1
107+
108+
- name: Upload test artifacts (node20 supported)
109+
if: steps.node20.outputs.supported == 'true' && steps.test.outcome == 'failure'
110+
uses: actions/upload-artifact@v4
111+
with:
112+
name: Test logs ${{ steps.artifact-names.outputs.name }}
113+
path: tests/**/logs/*.log*
114+
if-no-files-found: ignore
115+
- name: Upload test artifacts (node20 unsupported)
116+
if: steps.node20.outputs.supported == 'false' && steps.test.outcome == 'failure'
117+
uses: actions/upload-artifact@v4
118+
with:
119+
name: Test logs ${{ steps.artifact-names.outputs.name }}
120+
path: tests/**/logs/*.log*
121+
if-no-files-found: ignore
122+
123+
- name: Fail flow if tests failed
124+
shell: bash
125+
if: steps.test.outcome == 'failure'
126+
run: exit 1

‎.github/actions/setup-env/action.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -65,4 +65,4 @@ runs:
6565
id: set-redis-ref
6666
run: |
6767
export REDIS_REF="${{ inputs.redis-ref || '7.4'}}"
68-
echo "REDIS_REF=${REDIS_REF}" >> $GITHUB_OUTPUT
68+
echo "REDIS_REF=${REDIS_REF}" >> $GITHUB_OUTPUT

‎.github/actions/upload-artifacts-to-s3-without-make/action.yml

+32-24
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,6 @@ inputs:
99
aws-secret-access-key:
1010
description: 'AWS Secret Access Key'
1111
required: true
12-
github-ref:
13-
description: 'GitHub ref'
14-
required: true
1512
osnick:
1613
description: 'OS Nickname'
1714
required: false
@@ -20,29 +17,40 @@ inputs:
2017
runs:
2118
using: composite
2219
steps:
23-
- name: Configure AWS credentials
24-
uses: aws-actions/configure-aws-credentials@v3
25-
with: # todo: use role instead of access key
26-
aws-access-key-id: ${{ inputs.aws-access-key-id }}
27-
aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
28-
aws-region: "us-east-1"
29-
- name: Upload artifacts to S3 - staging
20+
- name: Configure AWS credentials and upload artifcats # todo - use aws role instead
3021
shell: bash
3122
run: |
32-
echo ::group::install aws cli
23+
echo ::group::install aws cli
3324
python3 -m venv .aws-cli-venv && source .aws-cli-venv/bin/activate &&
3425
pip3 install --upgrade pip && pip3 install --no-cache-dir awscli && rm -rf /var/cache/apk/*
35-
echo ::endgroup::
36-
echo ::group::upload artifacts
37-
SNAPSHOT=1 SHOW=1 VERBOSE=1 ./sbin/upload-artifacts
38-
echo ::endgroup::
39-
echo ::group::upload staging release
40-
RELEASE=1 SHOW=1 STAGING=1 VERBOSE=1 ./sbin/upload-artifacts
41-
echo ::endgroup::
42-
43-
echo ::group::upload production release
44-
# todo: trigger this manually instead
45-
if [[ "${{ inputs.github-ref}}" != 'refs/heads/master' ]]; then
46-
RELEASE=1 SHOW=1 VERBOSE=1 ./sbin/upload-artifacts
26+
echo ::endgroup::
27+
28+
# Variables from the workflow
29+
export AWS_ACCESS_KEY_ID="${{ inputs.aws-access-key-id }}"
30+
export AWS_SECRET_ACCESS_KEY="${{ inputs.aws-secret-access-key }}"
31+
export AWS_REGION="us-east-1"
32+
# Check if the required environment variables are set
33+
if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ] || [ -z "$AWS_REGION" ]; then
34+
echo "Missing AWS credentials or region configuration."
35+
exit 1
4736
fi
48-
echo ::endgroup::
37+
# Configure AWS CLI with provided credentials and region
38+
echo "Configuring AWS CLI with access keys..."
39+
aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID"
40+
aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY"
41+
aws configure set region "$AWS_REGION"
42+
43+
echo ::group::upload artifacts
44+
SNAPSHOT=1 SHOW=1 VERBOSE=1 ./sbin/upload-artifacts
45+
echo ::endgroup::
46+
echo ::group::upload staging release
47+
RELEASE=1 SHOW=1 STAGING=1 VERBOSE=1 ./sbin/upload-artifacts
48+
echo ::endgroup::
49+
50+
echo ::group::upload production release
51+
REF="${{ inputs.github-ref }}"
52+
PATTERN="refs/tags/v[0-9]+.*"
53+
if [[ $REF =~ $PATTERN ]]; then
54+
RELEASE=1 SHOW=1 VERBOSE=1 ./sbin/upload-artifacts
55+
fi
56+
echo ::endgroup::

0 commit comments

Comments
 (0)