Skip to content

Commit d2981c2

Browse files
rafierafie
authored
Build system modifications (RedisJSON#276)
- Added readies submodule - Refactored Makefile, added help - Renamed tests directory from test to tests, done some clean-ups - Converted Python tests from rmtest to RLTest - Moved automation from Python 2 to 3 - Refactored config.yml, added QA automation tests
1 parent 2ea20d0 commit d2981c2

File tree

138 files changed

+1447
-1234
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

138 files changed

+1447
-1234
lines changed

‎.circleci/config.yml

+92-64
Original file line numberDiff line numberDiff line change
@@ -14,110 +14,85 @@ commands:
1414
circleci step halt
1515
fi
1616
17-
jobs:
17+
setup-automation:
18+
steps:
19+
- run:
20+
name: Setup automation
21+
command: |
22+
git submodule update --init deps/readies
23+
./deps/readies/bin/getpy3
1824
25+
jobs:
1926
build:
2027
docker:
21-
- image: redisfab/rmbuilder:6.0.7-x64-centos7
22-
28+
- image: redisfab/rmbuilder:6.0.9-x64-centos7
2329
steps:
2430
- checkout
25-
2631
- run:
2732
name: Checkout submodules
2833
command: git submodule update --init --recursive
29-
3034
- run:
3135
name: Setup automation and install dependencies
3236
command: |
33-
(mkdir -p deps; cd deps; git clone https://github.com/RedisLabsModules/readies.git)
34-
PIP=1 FORCE=1 ./deps/readies/bin/getpy2
37+
./deps/readies/bin/getpy3
3538
./system-setup.py
36-
3739
- run:
3840
name: Install Redis
39-
command: ./deps/readies/bin/getredis -v 6
40-
41+
command: bash -l -c "./deps/readies/bin/getredis -v 6 --force"
4142
- run:
4243
name: Install Rust
4344
command: |
44-
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rust.sh
45-
sh rust.sh -y
46-
echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> ~/.bashrc
47-
source $HOME/.cargo/env
48-
rustc --version
49-
cargo --version
50-
rustup --version
51-
45+
./deps/readies/bin/getrust
46+
source $HOME/.cargo/env
47+
rustc --version
48+
cargo --version
49+
rustup --version
5250
- restore_cache:
5351
keys:
5452
- v2-dependencies-{{ arch }}-{{ checksum "Cargo.lock" }}
5553
# fallback to using the latest cache if no exact match is found
5654
- v2-dependencies-
57-
5855
- run:
5956
name: Check formatting
60-
command: |
61-
source $HOME/.cargo/env
62-
rustfmt --version
63-
cargo fmt -- --check
64-
57+
command: bash -l -c "make lint"
6558
- run:
6659
name: Build debug
67-
command: |
68-
source $HOME/.cargo/env
69-
cargo build --all --all-targets
70-
mv target/debug/librejson.so target/debug/rejson.so
71-
72-
- run:
73-
name: Run all tests
74-
command: |
75-
source $HOME/.cargo/env
76-
cargo test --features test --all
77-
python2 ./test/pytest/test.py
78-
60+
command: bash -l -c "make build DEBUG=1"
7961
- run:
8062
name: Build release
81-
command: |
82-
source $HOME/.cargo/env
83-
cargo build --all --all-targets --release
84-
mv target/release/librejson.so target/release/rejson.so
85-
63+
command: bash -l -c make
64+
- run:
65+
name: Run all tests
66+
command: bash -l -c "make pytest"
8667
- save_cache:
8768
key: v2-dependencies-{{ arch }}-{{ checksum "Cargo.lock" }}
8869
paths:
8970
- "~/.cargo"
9071
- "./target"
91-
9272
- early_return_for_forked_pull_requests
93-
9473
- run:
9574
name: Persist Artifacts
9675
command: |
9776
mkdir -p workspace/build
9877
cp target/release/$MODULE_ARTIFACT workspace/
9978
cp ramp.yml workspace/
100-
10179
- persist_to_workspace:
10280
root: workspace
10381
paths:
10482
- '*.so'
10583
- ramp.yml
10684
- build
10785

108-
10986
package_branch:
11087
docker:
111-
- image: redisfab/rmbuilder:6.0.7-x64-buster
88+
- image: redisfab/rmbuilder:6.0.9-x64-buster
11289
steps:
11390
- attach_workspace:
11491
at: /workspace
11592
- run:
11693
name: Package
117-
command: >-
118-
ramp pack -m /workspace/ramp.yml -o
119-
/workspace/build/$PACKAGE_NAME.{os}-{architecture}.$CIRCLE_BRANCH.zip
120-
/workspace/$MODULE_ARTIFACT
94+
command: |
95+
ramp pack -m /workspace/ramp.yml -o /workspace/build/$PACKAGE_NAME.{os}-{architecture}.$CIRCLE_BRANCH.zip /workspace/$MODULE_ARTIFACT
12196
- persist_to_workspace:
12297
root: /workspace
12398
paths:
@@ -127,22 +102,18 @@ jobs:
127102

128103
package_release:
129104
docker:
130-
- image: redisfab/rmbuilder:6.0.7-x64-buster
105+
- image: redisfab/rmbuilder:6.0.9-x64-buster
131106
steps:
132107
- attach_workspace:
133108
at: /workspace
134109
- run:
135110
name: Package
136-
command: >-
137-
ramp pack -m /workspace/ramp.yml -o
138-
/workspace/build/$PACKAGE_NAME.{os}-{architecture}.{semantic_version}.zip
139-
/workspace/$MODULE_ARTIFACT
111+
command: |
112+
ramp pack -m /workspace/ramp.yml -o /workspace/build/$PACKAGE_NAME.{os}-{architecture}.{semantic_version}.zip /workspace/$MODULE_ARTIFACT
140113
- run:
141114
name: Package
142-
command: >-
143-
ramp pack -m /workspace/ramp.yml -o
144-
/workspace/build/$PACKAGE_NAME.{os}-{architecture}.latest.zip
145-
/workspace/$MODULE_ARTIFACT
115+
command: |
116+
ramp pack -m /workspace/ramp.yml -o /workspace/build/$PACKAGE_NAME.{os}-{architecture}.latest.zip /workspace/$MODULE_ARTIFACT
146117
- persist_to_workspace:
147118
root: /workspace
148119
paths:
@@ -152,7 +123,7 @@ jobs:
152123

153124
deploy_branch:
154125
docker:
155-
- image: redisfab/rmbuilder:6.0.7-x64-buster
126+
- image: redisfab/rmbuilder:6.0.9-x64-buster
156127
steps:
157128
- attach_workspace:
158129
at: /workspace
@@ -163,7 +134,7 @@ jobs:
163134
164135
deploy_release:
165136
docker:
166-
- image: redisfab/rmbuilder:6.0.7-x64-buster
137+
- image: redisfab/rmbuilder:6.0.9-x64-buster
167138
steps:
168139
- attach_workspace:
169140
at: /workspace
@@ -172,6 +143,26 @@ jobs:
172143
command: |
173144
aws s3 cp /workspace/build/ s3://redismodules/$PACKAGE_NAME/ --acl public-read --recursive --exclude "*" --include "*.zip"
174145
146+
release_automation:
147+
docker:
148+
- image: redisfab/rmbuilder:6.0.9-x64-buster
149+
steps:
150+
- checkout
151+
- setup-automation
152+
- run:
153+
name: Run QA Automation
154+
command: MODULE_VERSION=$CIRCLE_TAG VERBOSE=1 TEST=release ./tests/qa/run
155+
156+
nightly_automation:
157+
docker:
158+
- image: redisfab/rmbuilder:6.0.9-x64-buster
159+
steps:
160+
- checkout
161+
- setup-automation
162+
- run:
163+
name: Run QA Automation
164+
command: MODULE_VERSION=$CIRCLE_BRANCH VERBOSE=1 TEST=nightly QUICK=1 ./tests/qa/run
165+
175166

176167
on-any-branch: &on-any-branch
177168
filters:
@@ -191,6 +182,26 @@ on-master: &on-master
191182
filters:
192183
branches:
193184
only: master
185+
tags:
186+
ignore: /.*/
187+
188+
on-integ-branch: &on-integ-branch
189+
filters:
190+
branches:
191+
only:
192+
- master
193+
- /^\d+\.\d+.*$/
194+
tags:
195+
ignore: /.*/
196+
197+
not-on-integ-branch: &not-on-integ-branch
198+
filters:
199+
branches:
200+
ignore:
201+
- master
202+
- /^\d+\.\d+.*$/
203+
tags:
204+
ignore: /.*/
194205

195206
on-version-tags: &on-version-tags
196207
filters:
@@ -199,11 +210,12 @@ on-version-tags: &on-version-tags
199210
tags:
200211
only: /^v[0-9].*/
201212

202-
on-master-and-version-tags: &on-master-and-version-tags
213+
on-integ-and-version-tags: &on-integ-and-version-tags
203214
filters:
204215
branches:
205216
only:
206217
- master
218+
- /^\d+\.\d+.*$/
207219
tags:
208220
only: /^v[0-9].*/
209221

@@ -215,18 +227,34 @@ workflows:
215227
- build:
216228
<<: *on-any-branch
217229
- package_branch:
218-
<<: *on-master
230+
<<: *on-integ-branch
219231
requires:
220232
- build
221233
- package_release:
222234
<<: *on-version-tags
223235
requires:
224236
- build
225237
- deploy_branch:
226-
<<: *on-master
238+
context: common
239+
<<: *on-integ-branch
227240
requires:
228241
- package_branch
229242
- deploy_release:
243+
context: common
230244
<<: *on-version-tags
231245
requires:
232246
- package_release
247+
- release_automation:
248+
<<: *on-version-tags
249+
requires:
250+
- deploy_release
251+
252+
nightly:
253+
triggers:
254+
- schedule:
255+
cron: "07 20 * * *"
256+
filters:
257+
branches:
258+
only: master
259+
jobs:
260+
- nightly_automation

‎.dockerignore

+5-2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
/target/
2+
/tests/pytest/logs/
3+
14
# Object files
25
*.o
36
*.ko
@@ -37,6 +40,6 @@
3740
*.pyc
3841

3942
# mkdocs
40-
site/
43+
/site/
4144

42-
.vscode/
45+
.vscode/

‎.gitignore

+8-18
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,28 @@
1-
# Cargo
21
/target/
2+
/tests/pytest/logs/
33

44
# These are backup files generated by rustfmt
55
**/*.rs.bk
66

7-
# IDEs
8-
.idea
9-
.project
10-
.vscode
11-
12-
# Eclipse
13-
.cproject
14-
.project
15-
.settings/
16-
177
# Python
188
*.pyc
199

2010
# Redis
21-
*.rdb
11+
# *.rdb
2212

2313
# mkdocs
24-
site/
14+
/site/
2515

2616
# packaging
27-
build/
17+
# build/
2818

2919
# Mac
3020
.DS_Store
3121

32-
# Eclipse
22+
# IDEs
23+
.idea
24+
.project
25+
.vscode
3326
.cproject
3427
.project
3528
.settings/
36-
37-
# Rust build artifacts
38-
/target

‎.gitmodules

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[submodule "deps/readies"]
2+
path = deps/readies
3+
url = https://github.com/RedisLabsModules/readies.git

‎Dockerfile

+7-11
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,26 @@
1+
2+
#----------------------------------------------------------------------------------------------
13
FROM rust:latest as builder
24

35
ENV LIBDIR /usr/lib/redis/modules
46

5-
ADD . /REJSON
6-
WORKDIR /REJSON
7+
ADD . /build
8+
WORKDIR /build
79

8-
# Set up a build environment
9-
RUN set -ex ;\
10-
mkdir -p deps ;\
11-
cd deps ;\
12-
git clone https://github.com/RedisLabsModules/readies.git
13-
RUN PIP=1 FORCE=1 ./deps/readies/bin/getpy2
10+
RUN ./deps/readies/bin/getpy3
1411
RUN ./system-setup.py
1512

16-
# Build the source
1713
RUN set -ex ;\
1814
cargo build --release ;\
1915
mv target/release/librejson.so target/release/rejson.so
2016

2117
#----------------------------------------------------------------------------------------------
22-
# Package the runner
2318
FROM redis:latest
2419

2520
ENV LIBDIR /usr/lib/redis/modules
2621
WORKDIR /data
22+
2723
RUN mkdir -p "$LIBDIR"
28-
COPY --from=builder /REJSON/target/release/rejson.so "$LIBDIR"
24+
COPY --from=builder /build/target/release/rejson.so "$LIBDIR"
2925

3026
CMD ["redis-server", "--loadmodule", "/usr/lib/redis/modules/rejson.so"]

0 commit comments

Comments
 (0)