Skip to content

Commit a09fc09

Browse files
kelvichkelvich
committed
pg_shard test suite
1 parent 316e377 commit a09fc09

File tree

3 files changed

+365
-0
lines changed

3 files changed

+365
-0
lines changed

‎contrib/pg_xtm/tests/pg_shard_transfers.go

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"sync"
6+
// "github.com/jackc/pgx"
7+
"pgx"
8+
)
9+
10+
const (
11+
TRANSFER_CONNECTIONS = 1
12+
INIT_AMOUNT = 10000
13+
N_ITERATIONS = 5000
14+
N_ACCOUNTS = TRANSFER_CONNECTIONS
15+
)
16+
17+
var cfg = pgx.ConnConfig{
18+
Host: "127.0.0.1",
19+
Port: 5433,
20+
// Database: "postgres",
21+
}
22+
23+
var running = false
24+
25+
func transfer(id int, wg *sync.WaitGroup) {
26+
var err error
27+
var conn *pgx.Conn
28+
29+
conn, err = pgx.Connect(cfg)
30+
checkErr(err)
31+
defer conn.Close()
32+
33+
for i:=0; i < N_ITERATIONS; i++ {
34+
exec(conn, "begin")
35+
exec(conn, "update t_10000 set v = v + 1 where u=3")
36+
exec(conn, "update t_10000 set v = v - 1 where u=4")
37+
exec(conn, "commit")
38+
}
39+
40+
wg.Done()
41+
}
42+
43+
func inspect(wg *sync.WaitGroup) {
44+
var sum int64
45+
var prevSum int64 = 0
46+
47+
conn, err := pgx.Connect(cfg)
48+
checkErr(err)
49+
50+
for running {
51+
sum = execQuery(conn, "select sum(v) from t_10000")
52+
if sum != prevSum {
53+
fmt.Println("Total = ", sum);
54+
prevSum = sum
55+
}
56+
}
57+
58+
conn.Close()
59+
wg.Done()
60+
}
61+
62+
func main() {
63+
// var transferWg sync.WaitGroup
64+
// var inspectWg sync.WaitGroup
65+
var err error
66+
var conn *pgx.Conn
67+
var s int64
68+
69+
conn, err = pgx.Connect(cfg)
70+
checkErr(err)
71+
defer conn.Close()
72+
73+
// err = conn.QueryRow("select sum(v) from t_10000").Scan(&s)
74+
// checkErr(err)
75+
76+
s = execQuery(conn, "select sum(v) from t_10000")
77+
fmt.Println(s)
78+
79+
80+
// transferWg.Add(TRANSFER_CONNECTIONS)
81+
// for i:=0; i<TRANSFER_CONNECTIONS; i++ {
82+
// go transfer(i, &transferWg)
83+
// }
84+
85+
// running = true
86+
// inspectWg.Add(1)
87+
// go inspect(&inspectWg)
88+
89+
// transferWg.Wait()
90+
91+
// running = false
92+
// inspectWg.Wait()
93+
94+
fmt.Printf("done\n")
95+
}
96+
97+
func exec(conn *pgx.Conn, stmt string) {
98+
var err error
99+
_, err = conn.Exec(stmt)
100+
checkErr(err)
101+
}
102+
103+
func execQuery(conn *pgx.Conn, stmt string) int64 {
104+
var err error
105+
var result int64
106+
// result, err = conn.SimpleQuery(stmt)
107+
// err = conn.QueryRow(stmt).Scan(&result)
108+
err = conn.SimpleQuery(stmt).Scan(&result)
109+
checkErr(err)
110+
return result
111+
}
112+
113+
func checkErr(err error) {
114+
if err != nil {
115+
panic(err)
116+
}
117+
}
118+

‎install_pg_shard.sh

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
#!/bin/sh
2+
3+
PG_SHARD_DIR=~/code/pg_shard_master
4+
PG_DIR=~/code/postgresql
5+
PG_XTM_DIR=$PG_DIR/contrib/pg_xtm
6+
7+
8+
########################################################################
9+
# Stop old stuff
10+
########################################################################
11+
./install/bin/pg_ctl -D ./install/data1 stop
12+
./install/bin/pg_ctl -D ./install/data2 stop
13+
./install/bin/pg_ctl -D ./install/data3 stop
14+
killall dtmd
15+
rm -rf install
16+
17+
18+
########################################################################
19+
# Build and run dtm and postgres
20+
########################################################################
21+
make install # assuming configured with --prefix=./install
22+
23+
cd $PG_SHARD_DIR
24+
make clean
25+
PATH=~/code/postgresql/install/bin/:$PATH make
26+
PATH=~/code/postgresql/install/bin/:$PATH make install
27+
28+
cd $PG_DIR
29+
./install/bin/initdb -D ./install/data1
30+
./install/bin/initdb -D ./install/data2
31+
./install/bin/initdb -D ./install/data3
32+
33+
34+
sed -i '' 's/#port =.*/port = 5433/' ./install/data2/postgresql.conf
35+
sed -i '' 's/#port =.*/port = 5434/' ./install/data3/postgresql.conf
36+
37+
sed -i '' "s/shared_preload_libraries =.*/shared_preload_libraries = 'pg_shard'/" ./install/data1/postgresql.conf
38+
sed -i '' "s/shared_preload_libraries =.*/shared_preload_libraries = ''/" ./install/data2/postgresql.conf
39+
sed -i '' "s/shared_preload_libraries =.*/shared_preload_libraries = ''/" ./install/data3/postgresql.conf
40+
41+
sed -i '' 's/#fsync =.*/fsync = off/' ./install/data1/postgresql.conf
42+
sed -i '' 's/#fsync =.*/fsync = off/' ./install/data2/postgresql.conf
43+
sed -i '' 's/#fsync =.*/fsync = off/' ./install/data3/postgresql.conf
44+
45+
46+
# echo 'pg_shard.use_dtm_transactions=1' >> ./install/data1/postgresql.conf
47+
# echo 'pg_shard.use_dtm_transactions=1' >> ./install/data2/postgresql.conf
48+
# echo 'pg_shard.use_dtm_transactions=1' >> ./install/data3/postgresql.conf
49+
50+
51+
52+
./install/bin/pg_ctl -D ./install/data1 -l ./install/data1/log start
53+
./install/bin/pg_ctl -D ./install/data2 -l ./install/data2/log start
54+
./install/bin/pg_ctl -D ./install/data3 -l ./install/data3/log start
55+
56+
57+
########################################################################
58+
# Configure pg_shard
59+
########################################################################
60+
61+
echo "127.0.0.1 5433" > ./install/data1/pg_worker_list.conf
62+
echo "127.0.0.1 5434" >> ./install/data1/pg_worker_list.conf
63+
64+
echo "127.0.0.1 5433" > ./install/data2/pg_worker_list.conf
65+
echo "127.0.0.1 5434" >> ./install/data2/pg_worker_list.conf
66+
67+
echo "127.0.0.1 5433" > ./install/data3/pg_worker_list.conf
68+
echo "127.0.0.1 5434" >> ./install/data3/pg_worker_list.conf
69+
70+
71+
./install/bin/createdb `whoami`
72+
./install/bin/createdb `whoami` -p5433
73+
./install/bin/createdb `whoami` -p5434
74+
75+
76+
# ./install/bin/psql -p 5433 << SQL
77+
# CREATE EXTENSION pg_dtm;
78+
# SQL
79+
80+
# ./install/bin/psql -p 5434 << SQL
81+
# CREATE EXTENSION pg_dtm;
82+
# SQL
83+
84+
./install/bin/psql << SQL
85+
86+
CREATE EXTENSION pg_shard;
87+
CREATE TABLE t(u int, v int);
88+
SELECT master_create_distributed_table(table_name := 't', partition_column := 'u');
89+
SELECT master_create_worker_shards(table_name := 't', shard_count := 8, replication_factor := 1);
90+
insert into t values(1,10000);
91+
insert into t values(2,10000);
92+
insert into t values(3,10000);
93+
insert into t values(4,10000);
94+
insert into t values(5,10000);
95+
insert into t values(6,10000);
96+
insert into t values(7,10000);
97+
insert into t values(8,10000);
98+
99+
SQL
100+
101+
102+
103+
104+
# insert into t (select generate_series(0,10), random()::integer);
105+
106+
# cd contrib/pg_xtm/dtmd
107+
# make clean
108+
# make
109+
# rm -rf /tmp/clog/*
110+
# ./bin/dtmd
111+
112+
113+
114+
115+
116+
117+

‎install_pg_shard_xtm.sh

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
#!/bin/sh
2+
3+
PG_SHARD_DIR=~/code/pg_shard
4+
PG_DIR=~/code/postgresql
5+
PG_XTM_DIR=$PG_DIR/contrib/pg_xtm
6+
7+
8+
########################################################################
9+
# Stop old stuff
10+
########################################################################
11+
./install/bin/pg_ctl -D ./install/data1 stop
12+
./install/bin/pg_ctl -D ./install/data2 stop
13+
./install/bin/pg_ctl -D ./install/data3 stop
14+
killall dtmd
15+
rm -rf install
16+
17+
18+
########################################################################
19+
# Build and run dtm and postgres
20+
########################################################################
21+
make install # assuming configured with --prefix=./install
22+
23+
cd $PG_SHARD_DIR
24+
make clean
25+
PATH=~/code/postgresql/install/bin/:$PATH make
26+
PATH=~/code/postgresql/install/bin/:$PATH make install
27+
28+
cd $PG_XTM_DIR
29+
make clean
30+
make
31+
make install
32+
33+
cd dtmd
34+
make clean
35+
make
36+
rm -rf /tmp/clog/*
37+
./bin/dtmd &
38+
39+
cd $PG_DIR
40+
41+
./install/bin/initdb -D ./install/data1
42+
./install/bin/initdb -D ./install/data2
43+
./install/bin/initdb -D ./install/data3
44+
45+
46+
sed -i '' 's/#port =.*/port = 5433/' ./install/data2/postgresql.conf
47+
sed -i '' 's/#port =.*/port = 5434/' ./install/data3/postgresql.conf
48+
49+
sed -i '' "s/shared_preload_libraries =.*/shared_preload_libraries = 'pg_dtm,pg_shard'/" ./install/data1/postgresql.conf
50+
sed -i '' "s/shared_preload_libraries =.*/shared_preload_libraries = 'pg_dtm,pg_shard'/" ./install/data2/postgresql.conf
51+
sed -i '' "s/shared_preload_libraries =.*/shared_preload_libraries = 'pg_dtm,pg_shard'/" ./install/data3/postgresql.conf
52+
53+
sed -i '' 's/#fsync =.*/fsync = off/' ./install/data1/postgresql.conf
54+
sed -i '' 's/#fsync =.*/fsync = off/' ./install/data2/postgresql.conf
55+
sed -i '' 's/#fsync =.*/fsync = off/' ./install/data3/postgresql.conf
56+
57+
58+
echo 'pg_shard.use_dtm_transactions=1' >> ./install/data1/postgresql.conf
59+
echo 'pg_shard.use_dtm_transactions=1' >> ./install/data2/postgresql.conf
60+
echo 'pg_shard.use_dtm_transactions=1' >> ./install/data3/postgresql.conf
61+
62+
63+
64+
./install/bin/pg_ctl -D ./install/data1 -l ./install/data1/log start
65+
./install/bin/pg_ctl -D ./install/data2 -l ./install/data2/log start
66+
./install/bin/pg_ctl -D ./install/data3 -l ./install/data3/log start
67+
68+
69+
########################################################################
70+
# Configure pg_shard
71+
########################################################################
72+
73+
echo "127.0.0.1 5433" > ./install/data1/pg_worker_list.conf
74+
echo "127.0.0.1 5434" >> ./install/data1/pg_worker_list.conf
75+
76+
echo "127.0.0.1 5433" > ./install/data2/pg_worker_list.conf
77+
echo "127.0.0.1 5434" >> ./install/data2/pg_worker_list.conf
78+
79+
echo "127.0.0.1 5433" > ./install/data3/pg_worker_list.conf
80+
echo "127.0.0.1 5434" >> ./install/data3/pg_worker_list.conf
81+
82+
83+
./install/bin/createdb `whoami`
84+
./install/bin/createdb `whoami` -p5433
85+
./install/bin/createdb `whoami` -p5434
86+
87+
88+
./install/bin/psql -p 5433 << SQL
89+
CREATE EXTENSION pg_dtm;
90+
SQL
91+
92+
./install/bin/psql -p 5434 << SQL
93+
CREATE EXTENSION pg_dtm;
94+
SQL
95+
96+
./install/bin/psql << SQL
97+
98+
CREATE EXTENSION pg_dtm;
99+
CREATE EXTENSION pg_shard;
100+
CREATE TABLE t(u int, v int);
101+
SELECT master_create_distributed_table(table_name := 't', partition_column := 'u');
102+
SELECT master_create_worker_shards(table_name := 't', shard_count := 8, replication_factor := 1);
103+
insert into t values(1,10000);
104+
insert into t values(2,10000);
105+
insert into t values(3,10000);
106+
insert into t values(4,10000);
107+
insert into t values(5,10000);
108+
insert into t values(6,10000);
109+
insert into t values(7,10000);
110+
insert into t values(8,10000);
111+
112+
SQL
113+
114+
115+
116+
117+
# insert into t (select generate_series(0,10), random()::integer);
118+
119+
# cd contrib/pg_xtm/dtmd
120+
# make clean
121+
# make
122+
# rm -rf /tmp/clog/*
123+
# ./bin/dtmd
124+
125+
126+
127+
128+
129+
130+

0 commit comments

Comments
 (0)