Skip to content

Commit 3ea5937

Browse files
rafiegkorland
rafie
and
gkorland
authored
Upgraded flow test infrastructure (RedisJSON#1008)
* Upgraded flow test infrastructure * upgrade redismodule-rs version * fixes 1 * Fix chain ranges Adapt chain of ranges for python3 --------- Co-authored-by: Guy Korland <gkorland@gmail.com>
1 parent 68f0200 commit 3ea5937

File tree

11 files changed

+629
-175
lines changed

11 files changed

+629
-175
lines changed

‎.circleci/config.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -440,7 +440,7 @@ jobs:
440440

441441
sanitize:
442442
docker:
443-
- image: redisfab/clang:13-x64-bullseye
443+
- image: redisfab/clang:16-x64-focal
444444
parameters:
445445
san-type:
446446
type: string
@@ -451,7 +451,7 @@ jobs:
451451
- run:
452452
name: Build & test
453453
shell: /bin/bash -l -eo pipefail
454-
command: make SAN=<<parameters.san-type>> build test SHOW=1 VERBOSE=1
454+
command: make SAN=<<parameters.san-type>> build test SHOW=1
455455
no_output_timeout: 30m
456456
- save-tests-logs
457457

@@ -672,7 +672,7 @@ workflows:
672672
name: build-with-redis-<<matrix.redis_version>>
673673
matrix:
674674
parameters:
675-
redis_version: ["6.0", "6.2", "7.0", "7.2-rc1", "unstable"]
675+
redis_version: ["6.0", "6.2", "7.0", "7.2-rc2", "unstable"]
676676

677677
nightly-perf-once-a-week:
678678
triggers:

‎Cargo.lock

Lines changed: 30 additions & 30 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Makefile

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,9 @@ make pytest # run flow tests using RLTest
2828
AOF=1 # run AOF persistency tests on a standalone Redis topology
2929
SLAVES=1 # run replication tests on standalone Redis topology
3030
CLUSTER=1 # run general tests on a OSS Redis Cluster topology
31-
VALGRIND|VG=1 # run specified tests with Valgrind
3231
VERBOSE=1 # display more RLTest-related information
32+
SAN=type # use LLVM sanitizer (type=address|memory|leak|thread)
33+
VG=1 # run specified tests with Valgrind
3334

3435
make bench # run benchmarks
3536

@@ -268,7 +269,7 @@ endif
268269

269270
sanbox:
270271
@docker run -it -v $(PWD):/rejson -w /rejson --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \
271-
$(SANBOX_ARGS) redisfab/clang:13-x64-bullseye bash
272+
$(SANBOX_ARGS) redisfab/clang:16-x64-focal bash
272273

273274
.PHONY: sanbox
274275

‎deps/readies

‎sbin/memcheck-summary

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#!/bin/bash
2+
3+
PROGNAME="${BASH_SOURCE[0]}"
4+
HERE="$(cd "$(dirname "$PROGNAME")" &>/dev/null && pwd)"
5+
ROOT=$(cd $HERE/.. && pwd)
6+
export READIES=$ROOT/deps/readies
7+
. $READIES/shibumi/defs
8+
9+
cd $HERE
10+
11+
#----------------------------------------------------------------------------------------------
12+
13+
valgrind_check() {
14+
echo -n "${NOCOLOR}"
15+
if grep -l "$1" $logdir/*.valgrind.log &> /dev/null; then
16+
echo
17+
echo "${LIGHTRED}### Valgrind: ${TYPE} detected:${RED}"
18+
grep -l "$1" $logdir/*.valgrind.log
19+
echo -n "${NOCOLOR}"
20+
E=1
21+
fi
22+
}
23+
24+
valgrind_summary() {
25+
local logdir="$ROOT/tests/$DIR/logs"
26+
27+
local leaks_head=0
28+
for file in $(ls $logdir/*.valgrind.log 2>/dev/null); do
29+
# If the last "definitely lost: " line of a logfile has a nonzero value, print the file name
30+
if tac "$file" | grep -a -m 1 "definitely lost: " | grep "definitely lost: [1-9][0-9,]* bytes" &> /dev/null; then
31+
if [[ $leaks_head == 0 ]]; then
32+
echo
33+
echo "${LIGHTRED}### Valgrind: leaks detected:${RED}"
34+
leaks_head=1
35+
fi
36+
echo "$file"
37+
E=1
38+
fi
39+
done
40+
41+
TYPE="invalid reads" valgrind_check "Invalid read"
42+
TYPE="invalid writes" valgrind_check "Invalid write"
43+
}
44+
45+
#----------------------------------------------------------------------------------------------
46+
47+
sanitizer_check() {
48+
if grep -l "$1" $logdir/*.asan.log* &> /dev/null; then
49+
echo
50+
echo "${LIGHTRED}### Sanitizer: ${TYPE} detected:${RED}"
51+
grep -l "$1" $logdir/*.asan.log*
52+
echo "${NOCOLOR}"
53+
if [[ -z $WARN || $WARN != 1 ]]; then
54+
E=1
55+
fi
56+
fi
57+
}
58+
59+
sanitizer_summary() {
60+
local logdir="$ROOT/tests/$DIR/logs"
61+
if ! TYPE="leaks" sanitizer_check "Direct leak"; then
62+
TYPE="leaks" sanitizer_check "detected memory leaks"
63+
fi
64+
TYPE="buffer overflow" sanitizer_check "dynamic-stack-buffer-overflow"
65+
TYPE="memory errors" sanitizer_check "memcpy-param-overlap"
66+
TYPE="stack use after scope" sanitizer_check "stack-use-after-scope"
67+
TYPE="use after free" sanitizer_check "heap-use-after-free"
68+
TYPE="signal 11" WARN=1 sanitizer_check "signal 11"
69+
}
70+
71+
#----------------------------------------------------------------------------------------------
72+
73+
E=0
74+
75+
DIRS=
76+
if [[ $UNIT == 1 ]]; then
77+
DIRS+=" ."
78+
fi
79+
if [[ $FLOW == 1 ]]; then
80+
DIRS+=" pytests"
81+
fi
82+
83+
if [[ $VG == 1 ]]; then
84+
for dir in $DIRS; do
85+
DIR="$dir" valgrind_summary
86+
done
87+
elif [[ -n $SAN ]]; then
88+
for dir in $DIRS; do
89+
DIR="$dir" sanitizer_summary
90+
done
91+
fi
92+
93+
if [[ $E == 0 ]]; then
94+
echo "# No leaks detected"
95+
fi
96+
97+
exit $E

‎sbin/system-setup.py

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -18,45 +18,45 @@ def __init__(self, args):
1818

1919
def common_first(self):
2020
self.install_downloaders()
21-
self.run("%s/bin/enable-utf8" % READIES, sudo=self.os != 'macos')
21+
self.run(f"{READIES}/bin/enable-utf8", sudo=self.os != 'macos')
2222
self.install("git unzip rsync")
2323

2424
if self.osnick == 'ol8':
2525
self.install("tar")
26-
self.run("%s/bin/getclang --modern" % READIES)
27-
self.run("%s/bin/getrust" % READIES)
28-
self.run("%s/bin/getcmake --usr" % READIES)
26+
self.run(f"{READIES}/bin/getclang --modern")
27+
self.run(f"{READIES}/bin/getrust")
28+
self.run(f"{READIES}/bin/getcmake --usr")
2929

3030
def debian_compat(self):
3131
self.install("python3-dev")
32-
self.run("%s/bin/getgcc" % READIES)
32+
self.run(f"{READIES}/bin/getgcc")
3333

3434
def redhat_compat(self):
3535
self.install("redhat-lsb-core")
3636
self.install("which")
37-
self.run("%s/bin/getgcc --modern" % READIES)
37+
self.run(f"{READIES}/bin/getgcc --modern")
3838

3939
if not self.platform.is_arm():
4040
self.install_linux_gnu_tar()
4141

4242
def fedora(self):
43-
self.run("%s/bin/getgcc" % READIES)
43+
self.run(f"{READIES}/bin/getgcc")
4444

4545
def macos(self):
4646
self.install_gnu_utils()
4747
self.install("binutils")
48-
# self.run("%s/bin/getgcc" % READIES)
49-
self.run("%s/bin/getclang --modern" % READIES)
48+
# self.run(f"{READIES}/bin/getgcc")
49+
self.run(f"{READIES}/bin/getclang --modern")
5050

5151
def common_last(self):
5252
if self.dist != "arch":
5353
self.install("lcov")
5454
else:
5555
self.install("lcov-git", aur=True)
56-
self.run("{PYTHON} {READIES}/bin/getrmpytools --reinstall --modern".format(PYTHON=self.python, READIES=READIES))
57-
self.pip_install("-r %s/tests/pytest/requirements.txt" % ROOT)
58-
self.run("%s/bin/getaws" % READIES)
59-
self.run("NO_PY2=1 %s/bin/getpudb" % READIES)
56+
self.run(f"{self.python} {READIES}/bin/getrmpytools --reinstall --modern")
57+
self.pip_install(f"-r {ROOT}/tests/pytest/requirements.txt")
58+
self.run(f"{READIES}/bin/getaws")
59+
self.run(f"NO_PY2=1 {READIES}/bin/getpudb")
6060

6161
#----------------------------------------------------------------------------------------------
6262

‎tests/memcheck/asan.supp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# leak:...

‎tests/memcheck/redis.san-ignorelist

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
fun:THPIsEnabled

0 commit comments

Comments
 (0)