Skip to content

Commit 78433e6

Browse files
agneumagneum
committed
Merge branch '162-custom-ci-test-runner' into 'master'
feat: create custom CI runner for tests (#162) Closes #162 See merge request postgres-ai/database-lab!239
2 parents e6baac3 + 179e1c7 commit 78433e6

File tree

4 files changed

+56
-22
lines changed

4 files changed

+56
-22
lines changed

‎.gitlab-ci.yml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ stages:
77
- test
88
- build-binary
99
- build-image
10+
- integration-test
1011

1112
test:
1213
stage: test
@@ -18,6 +19,7 @@ lint:
1819
script:
1920
- make lint
2021

22+
2123
.only_var_template: &only_tag_release
2224
only:
2325
variables:
@@ -219,3 +221,16 @@ build-image-swagger-latest:
219221
before_script:
220222
- export LATEST_TAG=$(echo ${CI_COMMIT_TAG%.*}-latest)
221223
- export TAGS="${DOCKER_NAME}:${LATEST_TAG}"
224+
225+
integration-test:
226+
<<: *only_feature
227+
stage: integration-test
228+
variables:
229+
IMAGE_TAG: "${CI_COMMIT_REF_SLUG}"
230+
script:
231+
- zfs list
232+
- bash test/1.synthetic.sh
233+
after_script:
234+
- bash test/_cleanup.sh
235+
tags:
236+
- dle-test

‎test/1.synthetic.sh

Lines changed: 33 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,45 +2,61 @@
22
set -euxo pipefail
33

44
DIR=${0%/*}
5-
IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:v2-0"
6-
POSTGRES_VERSION="${POSTGRES_VERSION:-10}"
5+
6+
IMAGE_TAG="${IMAGE_TAG:-"master"}"
7+
IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${IMAGE_TAG}"
8+
POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
9+
710
### Step 1. Prepare a machine with two disks, Docker and ZFS.
811

9-
source "${DIR}/_prerequisites.ubuntu.sh"
12+
#source "${DIR}/_prerequisites.ubuntu.sh"
1013
source "${DIR}/_zfs.file.sh"
1114

1215
### Step 2. Prepare database data directory.
1316
sudo docker run \
1417
--name dblab_pg_initdb \
1518
--label dblab_control \
16-
--env PGDATA=/var/lib/dblab/data \
19+
--label dblab_test \
20+
--env PGDATA=/var/lib/dle/test/data \
1721
--env POSTGRES_HOST_AUTH_METHOD=trust \
18-
--volume /var/lib/dblab:/var/lib/dblab \
22+
--volume /var/lib/dle/test:/var/lib/dle/test \
1923
--detach \
2024
postgres:${POSTGRES_VERSION}-alpine
2125

2226
for i in {1..300}; do
23-
sudo docker exec -it dblab_pg_initdb psql -U postgres -c 'select' > /dev/null 2>&1 && break || echo "test database is not ready yet"
27+
sudo docker exec dblab_pg_initdb psql -U postgres -c 'select' > /dev/null 2>&1 && break || echo "test database is not ready yet"
2428
sleep 1
2529
done
2630

2731
sleep 10
28-
sudo docker exec -it dblab_pg_initdb psql -U postgres -c 'create database test'
32+
sudo docker exec dblab_pg_initdb psql -U postgres -c 'create database test'
2933

3034
# 1,000,000 accounts, ~0.14 GiB of data.
31-
sudo docker exec -it dblab_pg_initdb pgbench -U postgres -i -s 10 test
35+
sudo docker exec dblab_pg_initdb pgbench -U postgres -i -s 10 test
3236

3337
sudo docker stop dblab_pg_initdb
3438
sudo docker rm dblab_pg_initdb
3539

40+
41+
3642
### Step 3. Configure and launch the Database Lab server.
3743
mkdir -p ~/.dblab
3844
cp ./configs/config.example.physical_generic.yml ~/.dblab/server_test.yml
3945
sed -ri 's/^(\s*)(host:.*$)/\1host: ""/' ~/.dblab/server_test.yml
4046
sed -ri 's/^(\s*)(port: 2345$)/\1port: 12345/' ~/.dblab/server_test.yml
47+
sed -ri 's/^(\s*)(from: 6000$)/\1from: 16000/' ~/.dblab/server_test.yml
48+
sed -ri 's/^(\s*)(mountDir: "\/var\/lib\/dblab"$)/\1mountDir: "\/var\/lib\/dle\/test"/' ~/.dblab/server_test.yml
49+
sed -ri 's/^(\s*)(clonesMountDir: \/var\/lib\/dblab\/clones$)/\1clonesMountDir: "\/var\/lib\/dle\/test\/clones"/' ~/.dblab/server_test.yml
50+
sed -ri 's/^(\s*)(unixSocketDir: \/var\/lib\/dblab\/sockets$)/\1unixSocketDir: "\/var\/lib\/dle\/test\/sockets"/' ~/.dblab/server_test.yml
51+
sed -ri 's/^(\s*)(to: 6100$)/\1to: 16100/' ~/.dblab/server_test.yml
4152
sed -ri 's/^(\s*)(debug:.*$)/\1debug: true/' ~/.dblab/server_test.yml
4253
sed -ri 's/^(\s*)(pool:.*$)/\1pool: "test_pool"/' ~/.dblab/server_test.yml
54+
sed -ri 's/^(\s*)(enabled: true$)/\1enabled: false/' ~/.dblab/server_test.yml
4355
sed -ri 's/^(\s*)(- physicalRestore$)/\1/' ~/.dblab/server_test.yml
56+
sed -ri 's/^(\s*)(PGUSER: "postgres"$)/\1/' ~/.dblab/server_test.yml
57+
sed -ri 's/^(\s*)(PGPASSWORD: "postgres"$)/\1/' ~/.dblab/server_test.yml
58+
sed -ri 's/^(\s*)(PGHOST: "source.hostname"$)/\1/' ~/.dblab/server_test.yml
59+
sed -ri 's/^(\s*)(PGPORT: 5432$)/\1/' ~/.dblab/server_test.yml
4460

4561
# replace postgres version
4662
sed -ri "s/:12/:${POSTGRES_VERSION}/g" ~/.dblab/server_test.yml
@@ -49,10 +65,11 @@ sudo docker run \
4965
--detach \
5066
--name dblab_test \
5167
--label dblab_control \
68+
--label dblab_test \
5269
--privileged \
5370
--publish 12345:12345 \
5471
--volume /var/run/docker.sock:/var/run/docker.sock \
55-
--volume /var/lib/dblab:/var/lib/dblab:rshared \
72+
--volume /var/lib/dle/test:/var/lib/dle/test:rshared \
5673
--volume ~/.dblab/server_test.yml:/home/dblab/configs/config.yml \
5774
"${IMAGE2TEST}"
5875

@@ -65,7 +82,7 @@ for i in {1..30}; do
6582
done
6683

6784
### Step 4. Setup Database Lab CLI.
68-
curl https://gitlab.com/postgres-ai/database-lab/-/raw/master/scripts/cli_install.sh | bash
85+
dblab --version > /dev/null 2>&1 || curl https://gitlab.com/postgres-ai/database-lab/-/raw/master/scripts/cli_install.sh | bash
6986
dblab --version
7087
dblab init --url http://localhost:12345 --token secret_token --environment-id test
7188
dblab instance status
@@ -74,18 +91,20 @@ dblab instance status
7491
dblab clone create --username testuser --password testuser --id testclone
7592
dblab clone list
7693
export PGPASSWORD=testuser
77-
psql "host=localhost port=6000 user=testuser dbname=test" -c '\l'
94+
psql "host=localhost port=16000 user=testuser dbname=test" -c '\l'
7895

7996
### Step 6. Reset clone
80-
psql "host=localhost port=6000 user=testuser dbname=test" -c 'create database reset_database';
81-
psql "host=localhost port=6000 user=testuser dbname=test" -c '\l'
97+
psql "host=localhost port=16000 user=testuser dbname=test" -c 'create database reset_database';
98+
psql "host=localhost port=16000 user=testuser dbname=test" -c '\l'
8299
dblab clone reset testclone
83100
dblab clone status testclone
84-
psql "host=localhost port=6000 user=testuser dbname=test" -c '\l'
101+
psql "host=localhost port=16000 user=testuser dbname=test" -c '\l'
85102
dblab clone destroy testclone
86103

87104
### Step 7. Destroy clone
88105
dblab clone create --username testuser --password testuser --id testclone2
89106
dblab clone list
90107
dblab clone destroy testclone2
91108
dblab clone list
109+
110+
source "${DIR}/_cleanup.sh"

‎test/_cleanup.sh

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,16 @@ set -euxo pipefail
44
ZFS_FILE="$(pwd)/zfs_file"
55

66
# TODO: Remove all docker containers related to the Database Lab.
7-
sudo docker ps -a --filter 'label=dblab_control' \
7+
sudo docker ps -a --filter 'label=dblab_test' \
88
| grep -v CONTAINER \
99
| awk '{print $1}' \
1010
| sudo xargs --no-run-if-empty docker rm -f \
1111
|| true
1212
sudo zpool destroy test_pool || true
13-
sudo rm -rf /var/lib/dblab/data/
14-
sudo umount /var/lib/dblab/data || true
13+
sudo rm -rf /var/lib/dle/test/data/
14+
sudo umount /var/lib/dle/test/data || true
1515
sudo rm -f "${ZFS_FILE}"
1616
sudo rm -f ~/.dblab/server_test.yml
17-
sudo rm -rf /var/lib/dblab/db.dump || true
18-
sudo rm -rf /var/lib/dblab/rds_db.dump || true
17+
sudo rm -rf /var/lib/dle/test/db.dump || true
18+
sudo rm -rf /var/lib/dle/test/rds_db.dump || true
1919
dblab config remove test || true

‎test/_zfs.file.sh

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,11 @@ sudo zpool create -f \
1010
-O atime=off \
1111
-O recordsize=8k \
1212
-O logbias=throughput \
13-
-m /var/lib/dblab \
13+
-m /var/lib/dle/test \
1414
test_pool \
1515
"${ZFS_FILE}"
1616

17-
sudo mkdir -p /var/lib/dblab/data
18-
sudo chmod 0755 /var/lib/dblab/data
17+
sudo mkdir -p /var/lib/dle/test/data
18+
sudo chmod 0755 /var/lib/dle/test/data
1919

2020
zfs list

0 commit comments

Comments
 (0)