Skip to content

Commit 5cb9c90

Browse files
knizhnikknizhnik
committed
2 parents a52572c + c864a2a commit 5cb9c90

File tree

31 files changed

+3191
-47
lines changed

31 files changed

+3191
-47
lines changed

‎contrib/pg_dtm/pg_dtm.c

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -352,8 +352,9 @@ DtmGetNewTransactionId(bool isSubXact)
352352
TransactionId xid;
353353

354354
XTM_INFO("%d: GetNewTransactionId\n", getpid());
355-
Assert(!DtmGlobalXidAssigned); /* We should not assign new Xid if we do not use previous one */
356-
355+
if (DtmGlobalXidAssigned) { /* We should not assign new Xid if we do not use previous one */
356+
elog(ERROR, "dtm_begin/join_transaction should be called prior to begin of global transaction");
357+
}
357358
/*
358359
* Workers synchronize transaction state at the beginning of each parallel
359360
* operation, so we can't account for new XIDs after that point.
@@ -856,12 +857,16 @@ dtm_get_current_snapshot_xcnt(PG_FUNCTION_ARGS)
856857
Datum
857858
dtm_begin_transaction(PG_FUNCTION_ARGS)
858859
{
859-
Assert(!TransactionIdIsValid(DtmNextXid));
860+
if (TransactionIdIsValid(DtmNextXid)) {
861+
elog(ERROR, "dtm_begin/join_transaction should be called only once for global transaction");
862+
}
860863
if (dtm == NULL) {
861864
elog(ERROR, "DTM is not properly initialized, please check that pg_dtm plugin was added to shared_preload_libraries list in postgresql.conf");
862865
}
863866
DtmNextXid = DtmGlobalStartTransaction(&DtmSnapshot, &dtm->minXid);
864-
Assert(TransactionIdIsValid(DtmNextXid));
867+
if (!TransactionIdIsValid(DtmNextXid)) {
868+
elog(ERROR, "Arbiter was not able to assign XID");
869+
}
865870
XTM_INFO("%d: Start global transaction %d, dtm->minXid=%d\n", getpid(), DtmNextXid, dtm->minXid);
866871

867872
DtmHasGlobalSnapshot = true;
@@ -873,9 +878,13 @@ dtm_begin_transaction(PG_FUNCTION_ARGS)
873878

874879
Datum dtm_join_transaction(PG_FUNCTION_ARGS)
875880
{
876-
Assert(!TransactionIdIsValid(DtmNextXid));
881+
if (TransactionIdIsValid(DtmNextXid)) {
882+
elog(ERROR, "dtm_begin/join_transaction should be called only once for global transaction");
883+
}
877884
DtmNextXid = PG_GETARG_INT32(0);
878-
Assert(TransactionIdIsValid(DtmNextXid));
885+
if (!TransactionIdIsValid(DtmNextXid)) {
886+
elog(ERROR, "Arbiter was not able to assign XID");
887+
}
879888

880889
DtmGlobalGetSnapshot(DtmNextXid, &DtmSnapshot, &dtm->minXid);
881890
XTM_INFO("%d: Join global transaction %d, dtm->minXid=%d\n", getpid(), DtmNextXid, dtm->minXid);

‎contrib/pg_dtm/tests/deploy_layouts/cluster.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,7 @@
1010
pg_port: 15432
1111
deploy_postgres: true
1212
pg_dtm_enable: true
13+
pg_config_role:
14+
- line: "dtm.buffer_size = 65536"
1315
pg_dtm_host: "{{ groups['dtm'][0] }}"
1416

‎contrib/pg_dtm/tests/deploy_layouts/cluster_pg_shard.yml

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,32 @@
11
---
2-
- hosts: master-workers
2+
3+
- hosts: dtm
34
roles:
45
- role: postgrespro
56
deploy_dtm: true
67

7-
- hosts: master-workers:workers
8+
- hosts: masters
89
roles:
910
- role: postgrespro
11+
pg_src: ./postgrespro_pgshard
12+
pg_version: xtm_pgshard
1013
pg_port: 15432
1114
deploy_postgres: true
15+
pg_dtm_enable: true
16+
pg_dtm_host: "{{ groups['dtm'][0] }}"
1217
deploy_pg_shard: true
18+
19+
- hosts: workers
20+
roles:
21+
- role: postgrespro
22+
pg_src: ./postgrespro_pgshard
23+
pg_version: xtm_pgshard
24+
pg_port: 15432
25+
deploy_postgres: true
1326
pg_dtm_enable: true
14-
pg_dtm_host: "{{ groups['master-workers'][0] }}"
27+
pg_dtm_host: "{{ groups['dtm'][0] }}"
28+
29+
- hosts: clients
30+
roles:
31+
- role: postgrespro
1532

‎contrib/pg_dtm/tests/deploy_layouts/roles/postgrespro/tasks/dtm.yml

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,19 @@
11
---
22

3-
- name: ensure we have checked out libuv-1.7.5
4-
git: repo=https://github.com/libuv/libuv.git
5-
dest={{libuv.src}}
6-
version={{libuv.version}}
7-
update=no
3+
# - name: ensure we have checked out libuv-1.7.5
4+
# git: repo=https://github.com/libuv/libuv.git
5+
# dest={{libuv.src}}
6+
# version={{libuv.version}}
7+
# update=no
88

9-
- name: build libuv
10-
shell: ./autogen.sh && ./configure --disable-shared --prefix={{libuv.dst}} && make && make install
11-
args:
12-
chdir: "{{libuv.src}}"
13-
creates: "{{libuv.dst}}/lib/libuv.a"
9+
# - name: build libuv
10+
# shell: ./autogen.sh && ./configure --disable-shared --prefix={{libuv.dst}} && make && make install
11+
# args:
12+
# chdir: "{{libuv.src}}"
13+
# creates: "{{libuv.dst}}/lib/libuv.a"
1414

1515
- name: compile dtmd
16-
shell: make "LIBUV_PREFIX={{libuv.dst}}"
16+
shell: "make clean && make"
1717
args:
1818
chdir: "{{pg_src}}/contrib/pg_dtm/dtmd"
1919
creates: "{{pg_src}}/contrib/pg_dtm/dtmd/bin/dtmd"

‎contrib/pg_dtm/tests/deploy_layouts/roles/postgrespro/tasks/main.yml

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,3 @@
5656

5757
- include: dtm.yml
5858
when: deploy_dtm
59-
60-
- include: pg_shard.yml
61-
when: deploy_pg_shard

‎contrib/pg_dtm/tests/deploy_layouts/roles/postgrespro/tasks/pg_shard.yml

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,50 @@
11
- name: checkout pg_shard
2+
# git: repo=https://github.com/citusdata/pg_shard.git
23
git: repo=git@gitlab.postgrespro.ru:s.kelvich/pg_shard.git
34
dest=./pg_shard
45
version=transaction_manager_integration
56
force=yes
67
update=yes
7-
key_file=.ssh/banktest
8+
key_file=.ssh/ppg-deploy
89
accept_hostkey=yes
910
depth=1
1011
register: pg_shard_source
1112

1213
- name: build pg_shard extension
13-
shell: "PATH={{ppg.dst}}/bin:$PATH make clean && PATH={{ppg.dst}}/bin:$PATH make && PATH={{ppg.dst}}/bin:$PATH make install"
14+
shell: "PATH={{pg_dst}}/bin:$PATH make clean && PATH={{pg_dst}}/bin:$PATH make && PATH={{pg_dst}}/bin:$PATH make install"
1415
args:
1516
chdir: "~/pg_shard"
16-
creates: "{{ppg.dst}}/lib/pg_shard.so"
17+
creates: "{{pg_dst}}/lib/pg_shard.so"
1718

1819
- name: enable pg_shard extension in postgresql.conf with dtm
1920
lineinfile:
20-
dest: "{{ppg.datadir}}/postgresql.conf"
21+
dest: "{{pg_datadir}}/postgresql.conf"
2122
regexp: "{{item.regexp}}"
2223
line: "{{item.line}}"
2324
state: present
2425
with_items:
2526
- line: "shared_preload_libraries = 'pg_dtm, pg_shard'"
2627
regexp: "^shared_preload_libraries "
27-
when: ppg.usedtm
28+
- line: "pg_shard.all_modifications_commutative = 1"
29+
regexp: "^pg_shard.all_modifications_commutative "
30+
- line: "pg_shard.use_dtm_transactions = 1"
31+
regexp: "^pg_shard.use_dtm_transactions "
32+
when: pg_dtm_enable
2833

2934
- name: enable pg_shard extension in postgresql.conf without dtm
3035
lineinfile:
31-
dest: "{{ppg.datadir}}/postgresql.conf"
36+
dest: "{{pg_datadir}}/postgresql.conf"
3237
regexp: "{{item.regexp}}"
3338
line: "{{item.line}}"
3439
state: present
3540
with_items:
3641
- line: "shared_preload_libraries = 'pg_shard'"
3742
regexp: "^shared_preload_libraries "
38-
when: not ppg.usedtm
39-
43+
- line: "pg_shard.all_modifications_commutative = 1"
44+
regexp: "^pg_shard.all_modifications_commutative "
45+
when: not pg_dtm_enable
4046

47+
- name: create pg_worker_list.conf
48+
shell: echo "{{item}} 15432" >> {{pg_datadir}}/pg_worker_list.conf
49+
with_items: groups['workers']
4150

‎contrib/pg_dtm/tests/deploy_layouts/roles/postgrespro/tasks/postgres.yml

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,13 @@
3232
state: present
3333
with_items: "{{pg_config}}"
3434

35+
- name: configure2 postgres on datanodes
36+
lineinfile:
37+
dest: "{{pg_datadir}}/postgresql.conf"
38+
line: "{{item.line}}"
39+
state: present
40+
with_items: "{{pg_config_role}}"
41+
3542
- name: enable dtm extension on datanodes
3643
lineinfile:
3744
dest: "{{pg_datadir}}/postgresql.conf"
@@ -56,7 +63,6 @@
5663
- name: start postgrespro
5764
command: "{{pg_dst}}/bin/pg_ctl start -w -D {{pg_datadir}} -l {{pg_datadir}}/pg.log"
5865

59-
60-
61-
66+
- name: create pg_dtm extension
67+
command: "{{pg_dst}}/bin/psql -p {{pg_port}} postgres -c 'create extension pg_dtm;'"
6268

‎contrib/pg_dtm/tests/deploy_layouts/roles/postgrespro/vars/main.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ pg_config:
1818
- line: "listen_addresses = '*'"
1919
- line: "max_connections = 512"
2020
- line: "port = {{pg_port}}"
21+
pg_config_role:
22+
- line: "#pg_config_role"
2123
pg_src: ./postgrespro
2224
pg_dst: /tmp/postgrespro-build
2325
pg_datadir: ./postgrespro-data

‎contrib/pg_dtm/tests/farms/mephi

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,5 @@ blade8 ansible_ssh_host=85.143.113.31 ansible_ssh_port=4208
1212
[dtm]
1313
blade8 ansible_ssh_host=85.143.113.31 ansible_ssh_port=4208
1414

15-
15+
[master]
16+
blade7 ansible_ssh_host=85.143.113.31 ansible_ssh_port=4207

‎contrib/pg_dtm/tests/farms/sai

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
[clients]
2-
158.250.29.4 ansible_ssh_user=s.kelvich
32
158.250.29.5 ansible_ssh_user=s.kelvich
43

54
[workers]
6-
158.250.29.6 ansible_ssh_user=s.kelvich
5+
158.250.29.4 ansible_ssh_user=s.kelvich
76
158.250.29.8 ansible_ssh_user=s.kelvich
87
158.250.29.9 ansible_ssh_user=s.kelvich
8+
9+
[masters]
910
158.250.29.10 ansible_ssh_user=s.kelvich
1011

1112
[dtm]
12-
158.250.29.10 ansible_ssh_user=s.kelvich
13+
158.250.29.6 ansible_ssh_user=s.kelvich
14+
15+

‎contrib/pg_dtm/tests/pg_shard_transfers.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,8 @@ func transfer(id int, wg *sync.WaitGroup) {
5959

6060
for i:=0; i < N_ITERATIONS; i++ {
6161
amount := 1
62-
account1 := rand.Intn(N_ACCOUNTS)
63-
account2 := rand.Intn(N_ACCOUNTS)
62+
account1 := rand.Intn(N_ACCOUNTS-1)+1
63+
account2 := rand.Intn(N_ACCOUNTS-1)+1
6464
exec(conn, "begin")
6565
exec(conn, fmt.Sprintf("update t set v = v - %d where u=%d", amount, account1))
6666
exec(conn, fmt.Sprintf("update t set v = v + %d where u=%d", amount, account2))

‎contrib/pg_dtm/tests/transfers

6.31 MB
Binary file not shown.

‎contrib/pg_dtm/tests/transfers.linux

6.35 MB
Binary file not shown.

‎contrib/pg_dtm/tests/transfers.yml

Lines changed: 65 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,81 @@
11
---
22

3-
- hosts: clients
4-
gather_facts: no
5-
tasks:
6-
- name: copy transfers binary
7-
copy: src=transfers.linux dest=~/transfers mode=a+x
3+
# - hosts: 158.250.29.4
4+
# gather_facts: no
5+
# tasks:
6+
# # # - shell: echo "host={{item}} user=s.kelvich port=15432 dbname=postgres"
7+
# # register: source
8+
# # with_items: groups['workers']
9+
10+
# # - debug: var=source
11+
12+
# - set_fact:
13+
# connstr: "{{ groups['workers'] | ~'=' }}"
14+
15+
# - debug: var=connstr
16+
17+
# - name: copy transfers binary
18+
# copy: src=transfers.linux dest=~/transfers mode=a+x
19+
20+
# - hosts: blade3
21+
# gather_facts: no
22+
# tasks:
23+
# - name: setup the databases
24+
# shell: "./transfers -d 'host=158.250.29.10 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.9 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.8 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.6 user=s.kelvich port=15432 dbname=postgres' -f -g -m -w 140 -r 0"
25+
# register: transfers_result
26+
# - debug: var=transfers_result
27+
28+
# - hosts: blade3
29+
# gather_facts: no
30+
# tasks:
31+
# - name: run transfers
32+
# shell: "./transfers -d 'host=158.250.29.10 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.9 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.8 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.6 user=s.kelvich port=15432 dbname=postgres' -s -g -m -w 140 -u 14000 -r 0"
33+
# register: transfers_result
34+
# - debug: var=transfers_result
35+
36+
37+
# - hosts: blade3
38+
# gather_facts: no
39+
# tasks:
40+
# - name: setup the databases
41+
# shell: "./transfers -d 'host=blade8 user=stas port=15432 dbname=postgres' -d 'host=blade7 user=stas port=15432 dbname=postgres' -d 'host=blade6 user=stas port=15432 dbname=postgres' -d 'host=blade5 user=stas port=15432 dbname=postgres' -d 'host=blade4 user=stas port=15432 dbname=postgres' -f -g -w 200 -r 0"
42+
# register: transfers_result
43+
# - debug: var=transfers_result
44+
45+
# - hosts: blade3
46+
# gather_facts: no
47+
# tasks:
48+
# - name: run transfers
49+
# shell: "./transfers -d 'host=blade8 user=stas port=15432 dbname=postgres' -d 'host=blade7 user=stas port=15432 dbname=postgres' -d 'host=blade6 user=stas port=15432 dbname=postgres' -d 'host=blade5 user=stas port=15432 dbname=postgres' -d 'host=blade4 user=stas port=15432 dbname=postgres' -s -g -w 200 -u 2500 -r 0"
50+
# register: transfers_result
51+
# - debug: var=transfers_result
52+
853

954
- hosts: 158.250.29.4
1055
gather_facts: no
1156
tasks:
1257
- name: setup the databases
13-
shell: "./transfers -d 'host=158.250.29.10 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.9 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.8 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.6 user=s.kelvich port=15432 dbname=postgres' -f -g -m -w 128 -r 0"
58+
shell: "./transfers -d 'host=158.250.29.10 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.8 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.9 user=s.kelvich port=15432 dbname=postgres' -f -g -m -w 105 -r 0"
1459
register: transfers_result
1560
- debug: var=transfers_result
1661

1762
- hosts: clients
1863
gather_facts: no
1964
tasks:
2065
- name: run transfers
21-
shell: "./transfers -d 'host=158.250.29.10 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.9 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.8 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.6 user=s.kelvich port=15432 dbname=postgres' -s -g -m -w 128 -u 5000"
66+
shell: "./transfers -d 'host=158.250.29.10 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.8 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.9 user=s.kelvich port=15432 dbname=postgres' -s -g -m -w 105 -u 4000 -r 0"
2267
register: transfers_result
2368
- debug: var=transfers_result
69+
70+
71+
72+
73+
74+
75+
76+
77+
78+
79+
80+
81+

‎contrib/pg_tsdtm/Makefile

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
MODULE_big = pg_dtm
2+
OBJS = pg_dtm.o
3+
4+
EXTENSION = pg_dtm
5+
DATA = pg_dtm--1.0.sql
6+
7+
ifdef USE_PGXS
8+
PG_CONFIG = pg_config
9+
PGXS := $(shell $(PG_CONFIG) --pgxs)
10+
include $(PGXS)
11+
else
12+
subdir = contrib/pg_dtm
13+
top_builddir = ../..
14+
include $(top_builddir)/src/Makefile.global
15+
include $(top_srcdir)/contrib/contrib-global.mk
16+
endif

0 commit comments

Comments
 (0)