Skip to content

Commit a9a1a5d

Browse files
vitabaksvitabaks
committed
Merge branch 'fix-tests' into 'master'
Fix tests (#237) See merge request postgres-ai/database-lab!267
2 parents a462349 + 4f2298c commit a9a1a5d

12 files changed

+571
-555
lines changed

‎test/1.synthetic.sh

Lines changed: 84 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,110 +1,131 @@
11
#!/bin/bash
22
set -euxo pipefail
33

4-
DIR=${0%/*}
5-
6-
IMAGE_TAG="${IMAGE_TAG:-"master"}"
7-
IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${IMAGE_TAG}"
4+
TAG="${TAG:-"master"}"
5+
IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${TAG}"
86
POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
97

10-
### Step 1. Prepare a machine with two disks, Docker and ZFS.
8+
DIR=${0%/*}
119

12-
#source "${DIR}/_prerequisites.ubuntu.sh"
10+
### Step 1. Prepare a machine with disk, Docker, and ZFS
11+
source "${DIR}/_prerequisites.ubuntu.sh"
1312
source "${DIR}/_zfs.file.sh"
1413

15-
### Step 2. Prepare database data directory.
14+
15+
### Step 2. Configure and launch the Database Lab Engine
16+
17+
## Prepare database data directory.
1618
sudo docker run \
1719
--name dblab_pg_initdb \
18-
--label dblab_control \
19-
--label dblab_test \
20-
--env PGDATA=/var/lib/dle/test/data \
20+
--label dblab_sync \
21+
--env PGDATA=/var/lib/postgresql/pgdata \
2122
--env POSTGRES_HOST_AUTH_METHOD=trust \
22-
--volume /var/lib/dle/test:/var/lib/dle/test \
23+
--volume /var/lib/dblab/dblab_pool/data:/var/lib/postgresql/pgdata \
2324
--detach \
24-
postgres:${POSTGRES_VERSION}-alpine
25+
postgres:"${POSTGRES_VERSION}"-alpine
2526

2627
for i in {1..300}; do
27-
sudo docker exec dblab_pg_initdb psql -U postgres -c 'select' > /dev/null 2>&1 && break || echo "test database is not ready yet"
28+
sudo docker exec -it dblab_pg_initdb psql -U postgres -c 'select' > /dev/null 2>&1 && break || echo "test database is not ready yet"
2829
sleep 1
2930
done
3031

31-
sleep 10
32-
sudo docker exec dblab_pg_initdb psql -U postgres -c 'create database test'
32+
# Create the test database
33+
sudo docker exec -it dblab_pg_initdb psql -U postgres -c 'create database test'
3334

35+
# Generate data in the test database using pgbench
3436
# 1,000,000 accounts, ~0.14 GiB of data.
35-
sudo docker exec dblab_pg_initdb pgbench -U postgres -i -s 10 test
37+
sudo docker exec -it dblab_pg_initdb pgbench -U postgres -i -s 10 test
3638

39+
# Stop and remove the container
3740
sudo docker stop dblab_pg_initdb
3841
sudo docker rm dblab_pg_initdb
3942

40-
41-
42-
### Step 3. Configure and launch the Database Lab server.
43+
# Copy the contents of configuration example
4344
mkdir -p ~/.dblab
44-
cp ./configs/config.example.physical_generic.yml ~/.dblab/server_test.yml
45-
sed -ri 's/^(\s*)(host:.*$)/\1host: ""/' ~/.dblab/server_test.yml
46-
sed -ri 's/^(\s*)(port: 2345$)/\1port: 12345/' ~/.dblab/server_test.yml
47-
sed -ri 's/^(\s*)(from: 6000$)/\1from: 16000/' ~/.dblab/server_test.yml
48-
sed -ri 's/^(\s*)(mountDir: "\/var\/lib\/dblab"$)/\1mountDir: "\/var\/lib\/dle\/test"/' ~/.dblab/server_test.yml
49-
sed -ri 's/^(\s*)(clonesMountDir: \/var\/lib\/dblab\/clones$)/\1clonesMountDir: "\/var\/lib\/dle\/test\/clones"/' ~/.dblab/server_test.yml
50-
sed -ri 's/^(\s*)(unixSocketDir: \/var\/lib\/dblab\/sockets$)/\1unixSocketDir: "\/var\/lib\/dle\/test\/sockets"/' ~/.dblab/server_test.yml
51-
sed -ri 's/^(\s*)(to: 6100$)/\1to: 16100/' ~/.dblab/server_test.yml
52-
sed -ri 's/^(\s*)(debug:.*$)/\1debug: true/' ~/.dblab/server_test.yml
53-
sed -ri 's/^(\s*)(pool:.*$)/\1pool: "test_pool"/' ~/.dblab/server_test.yml
54-
sed -ri 's/^(\s*)(enabled: true$)/\1enabled: false/' ~/.dblab/server_test.yml
55-
sed -ri 's/^(\s*)(- physicalRestore$)/\1/' ~/.dblab/server_test.yml
56-
sed -ri 's/^(\s*)(PGUSER: "postgres"$)/\1/' ~/.dblab/server_test.yml
57-
sed -ri 's/^(\s*)(PGPASSWORD: "postgres"$)/\1/' ~/.dblab/server_test.yml
58-
sed -ri 's/^(\s*)(PGHOST: "source.hostname"$)/\1/' ~/.dblab/server_test.yml
59-
sed -ri 's/^(\s*)(PGPORT: 5432$)/\1/' ~/.dblab/server_test.yml
6045

46+
curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG}"/configs/config.example.logical_generic.yml \
47+
--output ~/.dblab/server.yml
48+
49+
# Edit the following options
50+
sed -ri 's/^(\s*)(debug:.*$)/\1debug: true/' ~/.dblab/server.yml
51+
sed -ri 's/^(\s*)(- logicalDump$)/\1/' ~/.dblab/server.yml
52+
sed -ri 's/^(\s*)(- logicalRestore$)/\1/' ~/.dblab/server.yml
6153
# replace postgres version
62-
sed -ri "s/:12/:${POSTGRES_VERSION}/g" ~/.dblab/server_test.yml
54+
sed -ri "s/:13/:${POSTGRES_VERSION}/g" ~/.dblab/server.yml
6355

56+
## Launch Database Lab server
6457
sudo docker run \
65-
--detach \
66-
--name dblab_test \
58+
--name dblab_server \
6759
--label dblab_control \
68-
--label dblab_test \
6960
--privileged \
70-
--publish 12345:12345 \
61+
--publish 2345:2345 \
7162
--volume /var/run/docker.sock:/var/run/docker.sock \
72-
--volume /var/lib/dle/test:/var/lib/dle/test:rshared \
73-
--volume ~/.dblab/server_test.yml:/home/dblab/configs/config.yml \
63+
--volume /var/lib/dblab/dblab_pool/dump:/var/lib/dblab/dblab_pool/dump \
64+
--volume /var/lib/dblab:/var/lib/dblab/:rshared \
65+
--volume ~/.dblab/server.yml:/home/dblab/configs/config.yml \
66+
--env DOCKER_API_VERSION=1.39 \
67+
--detach \
7468
"${IMAGE2TEST}"
7569

76-
sudo docker logs -f dblab_test 2>&1 | awk '{print "[CONTAINER dblab_test]: "$0}' &
70+
# Check the Database Lab Engine logs
71+
sudo docker logs dblab_server -f 2>&1 | awk '{print "[CONTAINER dblab_server]: "$0}' &
7772

7873
### Waiting for the Database Lab Engine initialization.
79-
for i in {1..30}; do
80-
curl http://localhost:12345 > /dev/null 2>&1 && break || echo "dblab is not ready yet"
81-
sleep 10
74+
for i in {1..300}; do
75+
curl http://localhost:2345 > /dev/null 2>&1 && break || echo "dblab is not ready yet"
76+
sleep 1
8277
done
8378

84-
### Step 4. Setup Database Lab CLI.
85-
dblab --version > /dev/null 2>&1 || curl https://gitlab.com/postgres-ai/database-lab/-/raw/master/scripts/cli_install.sh | bash
79+
80+
### Step 3. Start cloning
81+
82+
# Install Database Lab client CLI
83+
curl https://gitlab.com/postgres-ai/database-lab/-/raw/master/scripts/cli_install.sh | bash
84+
sudo mv ~/.dblab/dblab /usr/local/bin/dblab
85+
8686
dblab --version
87-
dblab init --url http://localhost:12345 --token secret_token --environment-id test
87+
88+
# Initialize CLI configuration
89+
dblab init \
90+
--environment-id=test \
91+
--url=http://localhost:2345 \
92+
--token=secret_token \
93+
--insecure
94+
95+
# Check the configuration by fetching the status of the instance:
8896
dblab instance status
8997

90-
### Step 5. Create a clone and connect to it.
91-
dblab clone create --username testuser --password testuser --id testclone
92-
dblab clone list
93-
export PGPASSWORD=testuser
94-
psql "host=localhost port=16000 user=testuser dbname=test" -c '\l'
9598

96-
### Step 6. Reset clone
97-
psql "host=localhost port=16000 user=testuser dbname=test" -c 'create database reset_database';
98-
psql "host=localhost port=16000 user=testuser dbname=test" -c '\l'
99+
## Create a clone
100+
dblab clone create \
101+
--username dblab_user_1 \
102+
--password secret_password \
103+
--id testclone
104+
105+
# Connect to a clone and check the available table
106+
PGPASSWORD=secret_password psql \
107+
"host=localhost port=6000 user=dblab_user_1 dbname=test" -c '\dt+'
108+
109+
# Drop table
110+
PGPASSWORD=secret_password psql \
111+
"host=localhost port=6000 user=dblab_user_1 dbname=test" -c 'drop table pgbench_accounts'
112+
113+
PGPASSWORD=secret_password psql \
114+
"host=localhost port=6000 user=dblab_user_1 dbname=test" -c '\dt+'
115+
116+
## Reset clone
99117
dblab clone reset testclone
118+
119+
# Check the status of the clone
100120
dblab clone status testclone
101-
psql "host=localhost port=16000 user=testuser dbname=test" -c '\l'
102-
dblab clone destroy testclone
103121

104-
### Step 7. Destroy clone
105-
dblab clone create --username testuser --password testuser --id testclone2
106-
dblab clone list
107-
dblab clone destroy testclone2
122+
# Check the database objects (everything should be the same as when we started)
123+
PGPASSWORD=secret_password psql \
124+
"host=localhost port=6000 user=dblab_user_1 dbname=test" -c '\dt+'
125+
126+
### Step 4. Destroy clone
127+
dblab clone destroy testclone
108128
dblab clone list
109129

130+
### Finish. clean up
110131
source "${DIR}/_cleanup.sh"

‎test/2.logical_generic.sh

Lines changed: 112 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,82 +1,143 @@
11
#!/bin/bash
22
set -euxo pipefail
33

4-
DIR=${0%/*}
5-
IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:master"
4+
TAG="${TAG:-"master"}"
5+
IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${TAG}"
66
SOURCE_DBNAME="${SOURCE_DBNAME:-test}"
7-
SOURCE_HOST="${SOURCE_HOST:-172.31.39.142}"
7+
SOURCE_HOST="${SOURCE_HOST:-172.17.0.1}"
8+
SOURCE_PORT="${SOURCE_PORT:-7432}"
89
SOURCE_USERNAME="${SOURCE_USERNAME:-postgres}"
9-
SOURCE_PASSWORD="${SOURCE_PASSWORD:-qwerty}"
10-
POSTGRES_VERSION="${POSTGRES_VERSION:-10}"
10+
SOURCE_PASSWORD="${SOURCE_PASSWORD:-secretpassword}"
11+
POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
12+
13+
DIR=${0%/*}
14+
15+
if [[ "${SOURCE_HOST}" = "172.17.0.1" ]]; then
16+
### Step 0. Create source database
17+
sudo rm -rf "$(pwd)"/postgresql/"${POSTGRES_VERSION}"/test || true
18+
sudo docker run \
19+
--name postgres"${POSTGRES_VERSION}" \
20+
--label pgdb \
21+
--privileged \
22+
--publish 172.17.0.1:"${SOURCE_PORT}":5432 \
23+
--env PGDATA=/var/lib/postgresql/pgdata \
24+
--env POSTGRES_USER="${SOURCE_USERNAME}" \
25+
--env POSTGRES_PASSWORD="${SOURCE_PASSWORD}" \
26+
--env POSTGRES_DB="${SOURCE_DBNAME}" \
27+
--env POSTGRES_HOST_AUTH_METHOD=md5 \
28+
--volume "$(pwd)"/postgresql/"${POSTGRES_VERSION}"/test:/var/lib/postgresql/pgdata \
29+
--detach \
30+
postgres:"${POSTGRES_VERSION}-alpine"
1131

12-
### Step 1: Prepare a machine with two disks, Docker and ZFS
32+
for i in {1..300}; do
33+
sudo docker exec -it postgres"${POSTGRES_VERSION}" psql -d "${SOURCE_DBNAME}" -U postgres -c 'select' > /dev/null 2>&1 && break || echo "test database is not ready yet"
34+
sleep 1
35+
done
1336

37+
# Generate data in the test database using pgbench
38+
# 1,000,000 accounts, ~0.14 GiB of data.
39+
sudo docker exec -it postgres"${POSTGRES_VERSION}" pgbench -U postgres -i -s 10 "${SOURCE_DBNAME}"
40+
41+
# Database info
42+
sudo docker exec -it postgres"${POSTGRES_VERSION}" psql -U postgres -c "\l+ ${SOURCE_DBNAME}"
43+
fi
44+
45+
### Step 1. Prepare a machine with disk, Docker, and ZFS
1446
source "${DIR}/_prerequisites.ubuntu.sh"
1547
source "${DIR}/_zfs.file.sh"
1648

17-
### Step 2. Prepare database data directory
1849

19-
### Step ?. Configure and launch the Database Lab server
50+
### Step 2. Configure and launch the Database Lab Engine
51+
52+
# Copy the contents of configuration example
2053
mkdir -p ~/.dblab
21-
cp ./configs/config.example.logical_generic.yml ~/.dblab/server_test.yml
22-
sed -ri 's/^(\s\s)(port:.*$)/\1port: 12345/' ~/.dblab/server_test.yml
23-
sed -ri 's/^(\s*)(debug:.*$)/\1debug: true/' ~/.dblab/server_test.yml
24-
sed -ri 's/^(\s*)(pool:.*$)/\1pool: "test_pool"/' ~/.dblab/server_test.yml
25-
sed -ri "s/^(\s*)(host: 34\.56\.78\.90$)/\1host: \"${SOURCE_HOST}\"/" ~/.dblab/server_test.yml
26-
sed -ri "s/^(\s*)(dbname:.*$)/\1dbname: \"${SOURCE_DBNAME}\"/" ~/.dblab/server_test.yml
27-
sed -ri "s/^(\s*)(username: postgres$)/\1username: \"${SOURCE_USERNAME}\"/" ~/.dblab/server_test.yml
28-
sed -ri "s/^(\s*)(password:.*$)/\1password: \"${SOURCE_PASSWORD}\"/" ~/.dblab/server_test.yml
29-
30-
### Step ?. Create source database
31-
export PGPASSWORD=${SOURCE_PASSWORD}
32-
psql -h ${SOURCE_HOST} -U ${SOURCE_USERNAME} -c 'drop database test' || echo "database test does not exist"
33-
psql -h ${SOURCE_HOST} -U ${SOURCE_USERNAME} -c 'create database test'
34-
pgbench -h ${SOURCE_HOST} -U ${SOURCE_USERNAME} -i -s 10 test
35-
36-
### Step ? Run dblab
54+
55+
curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG}"/configs/config.example.logical_generic.yml \
56+
--output ~/.dblab/server.yml
57+
58+
# Edit the following options
59+
sed -ri "s/^(\s*)(debug:.*$)/\1debug: true/" ~/.dblab/server.yml
60+
sed -ri "s/^(\s*)(dbname:.*$)/\1dbname: ${SOURCE_DBNAME}/" ~/.dblab/server.yml
61+
sed -ri "s/^(\s*)(host: 34.56.78.90$)/\1host: ${SOURCE_HOST}/" ~/.dblab/server.yml
62+
sed -ri "s/^(\s*)(port: 5432$)/\1port: ${SOURCE_PORT}/" ~/.dblab/server.yml
63+
sed -ri "s/^(\s*)(username: postgres$)/\1username: ${SOURCE_USERNAME}/" ~/.dblab/server.yml
64+
sed -ri "s/^(\s*)(password:.*$)/\1password: ${SOURCE_PASSWORD}/" ~/.dblab/server.yml
65+
# replace postgres version
66+
sed -ri "s/:13/:${POSTGRES_VERSION}/g" ~/.dblab/server.yml
67+
68+
## Launch Database Lab server
3769
sudo docker run \
38-
--detach \
39-
--name dblab_test \
70+
--name dblab_server \
4071
--label dblab_control \
4172
--privileged \
42-
--publish 12345:12345 \
73+
--publish 2345:2345 \
4374
--volume /var/run/docker.sock:/var/run/docker.sock \
44-
--volume /var/lib/dblab:/var/lib/dblab:rshared \
45-
--volume /var/lib/dblab/db.dump:/var/lib/dblab/db.dump \
46-
--volume ~/.dblab/server_test.yml:/home/dblab/configs/config.yml \
75+
--volume /var/lib/dblab/dblab_pool/dump:/var/lib/dblab/dblab_pool/dump \
76+
--volume /var/lib/dblab:/var/lib/dblab/:rshared \
77+
--volume ~/.dblab/server.yml:/home/dblab/configs/config.yml \
78+
--env DOCKER_API_VERSION=1.39 \
79+
--detach \
4780
"${IMAGE2TEST}"
4881

49-
sudo docker logs -f dblab_test 2>&1 | awk '{print "[CONTAINER dblab_test]: "$0}' &
82+
# Check the Database Lab Engine logs
83+
sudo docker logs dblab_server -f 2>&1 | awk '{print "[CONTAINER dblab_server]: "$0}' &
5084

51-
### Waiting fori dblab initialization
52-
for i in {1..30}; do
53-
curl http://localhost:12345 > /dev/null 2>&1 && break || echo "dblab is not ready yet"
54-
sleep 10
85+
### Waiting for the Database Lab Engine initialization.
86+
for i in {1..300}; do
87+
curl http://localhost:2345 > /dev/null 2>&1 && break || echo "dblab is not ready yet"
88+
sleep 1
5589
done
5690

57-
### Step ?. Setup Dnd init atabase Lab client CLI
91+
92+
### Step 3. Start cloning
93+
94+
# Install Database Lab client CLI
5895
curl https://gitlab.com/postgres-ai/database-lab/-/raw/master/scripts/cli_install.sh | bash
96+
sudo mv ~/.dblab/dblab /usr/local/bin/dblab
97+
5998
dblab --version
60-
dblab init --url http://localhost:12345 --token secret_token --environment-id test
99+
100+
# Initialize CLI configuration
101+
dblab init \
102+
--environment-id=test \
103+
--url=http://localhost:2345 \
104+
--token=secret_token \
105+
--insecure
106+
107+
# Check the configuration by fetching the status of the instance:
61108
dblab instance status
62109

63-
### Step ?. Create clone and connect to it
64-
dblab clone create --username testuser --password testuser --id testclone
65-
dblab clone list
66-
export PGPASSWORD=testuser
67-
psql "host=localhost port=6000 user=testuser dbname=test" -c '\l'
68110

111+
## Create a clone
112+
dblab clone create \
113+
--username dblab_user_1 \
114+
--password secret_password \
115+
--id testclone
116+
117+
# Connect to a clone and check the available table
118+
PGPASSWORD=secret_password psql \
119+
"host=localhost port=6000 user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c '\dt+'
120+
121+
# Drop table
122+
PGPASSWORD=secret_password psql \
123+
"host=localhost port=6000 user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c 'drop table pgbench_accounts'
124+
125+
PGPASSWORD=secret_password psql \
126+
"host=localhost port=6000 user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c '\dt+'
69127

70-
### Step 6. Reset clone
71-
psql "host=localhost port=6000 user=testuser dbname=test" -c 'create database reset_database';
72-
psql "host=localhost port=6000 user=testuser dbname=test" -c '\l'
128+
## Reset clone
73129
dblab clone reset testclone
130+
131+
# Check the status of the clone
74132
dblab clone status testclone
75-
psql "host=localhost port=6000 user=testuser dbname=test" -c '\l'
76-
dblab clone destroy testclone
77133

78-
### Step 7. Destroy clone
79-
dblab clone create --username testuser --password testuser --id testclone2
80-
dblab clone list
81-
dblab clone destroy testclone2
134+
# Check the database objects (everything should be the same as when we started)
135+
PGPASSWORD=secret_password psql \
136+
"host=localhost port=6000 user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c '\dt+'
137+
138+
### Step 4. Destroy clone
139+
dblab clone destroy testclone
82140
dblab clone list
141+
142+
### Finish. clean up
143+
source "${DIR}/_cleanup.sh"

0 commit comments

Comments
 (0)