Skip to content

Commit 37624c4

Browse files
knizhnikknizhnik
committed
Add 3PC support
2 parents 59e22f8 + 0e57e6b commit 37624c4

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+103
-5739
lines changed

‎.travis.yml

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,26 +5,24 @@ services:
55

66
before_script: env && pip3 install -r contrib/mmts/tests2/requirements.txt
77

8-
# script: cd contrib/mmts/tests2 && python3 test_regression.py
9-
script: cd contrib/mmts/tests2 && python3 test_recovery.py
8+
script: cd contrib/mmts/tests2 && python3 -m unittest discover
109

1110
notifications:
1211
email:
1312
- s.kelvich@postgrespro.ru
13+
- k.knizhnik@postgrespro.ru
1414

15+
## Old TAP-based tests:
1516

1617
# language: perl
1718
# os:
1819
# - linux
19-
# #- osx
2020
# compiler:
2121
# - gcc
2222
# - clang
2323
# install: cpanm IPC::Run DBD::Pg Proc::ProcessTable
2424
# before_script: ./configure --enable-cassert --enable-tap-tests && make -j4
2525
# env:
26-
# #- TESTDIR=.
27-
# #- TESTDIR=src/test/recovery
2826
# - TESTDIR=contrib/raftable
2927
# - TESTDIR=contrib/mmts
3028
# script: cd $TESTDIR && make check
@@ -33,4 +31,3 @@ notifications:
3331
# - s.kelvich@postgrespro.ru
3432
# - c.pan@postgrespro.ru
3533
# - k.knizhnik@postgrespro.ru
36-

‎contrib/mmts/.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
tests/node*
2+
tmp_check

‎contrib/mmts/Cluster.pm

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,10 @@ sub new
4040
foreach my $i (1..$nodenum)
4141
{
4242
my $host = "127.0.0.1";
43-
my ($pgport, $raftport) = allocate_ports($host, 2);
43+
my ($pgport, $arbiter_port) = allocate_ports($host, 2);
4444
my $node = new PostgresNode("node$i", $host, $pgport);
4545
$node->{id} = $i;
46-
$node->{raftport} = $raftport;
46+
$node->{arbiter_port} = $arbiter_port;
4747
push(@$nodes, $node);
4848
}
4949

@@ -71,16 +71,16 @@ sub configure
7171
{
7272
my ($self) = @_;
7373
my $nodes = $self->{nodes};
74+
my $nnodes = scalar @{ $nodes };
7475

75-
my $connstr = join(',', map { "${ \$_->connstr('postgres') }" } @$nodes);
76-
my $raftpeers = join(',', map { join(':', $_->{id}, $_->host, $_->{raftport}) } @$nodes);
76+
my $connstr = join(', ', map { "${ \$_->connstr('postgres') } arbiter_port=${ \$_->{arbiter_port} }" } @$nodes);
7777

7878
foreach my $node (@$nodes)
7979
{
8080
my $id = $node->{id};
8181
my $host = $node->host;
8282
my $pgport = $node->port;
83-
my $raftport = $node->{raftport};
83+
my $arbiter_port = $node->{arbiter_port};
8484

8585
$node->append_conf("postgresql.conf", qq(
8686
log_statement = none
@@ -94,20 +94,22 @@ sub configure
9494
fsync = off
9595
max_wal_senders = 10
9696
wal_sender_timeout = 0
97-
default_transaction_isolation = 'repeatable read'
97+
default_transaction_isolation = 'repeatable read'
9898
max_replication_slots = 10
99-
shared_preload_libraries = 'raftable,multimaster'
99+
shared_preload_libraries = 'multimaster'
100+
101+
multimaster.arbiter_port = $arbiter_port
100102
multimaster.workers = 10
101103
multimaster.queue_size = 10485760 # 10mb
102104
multimaster.node_id = $id
103105
multimaster.conn_strings = '$connstr'
104-
multimaster.use_raftable = false
105106
multimaster.heartbeat_recv_timeout = 1000
106107
multimaster.heartbeat_send_timeout = 250
107-
multimaster.max_nodes = 3
108+
multimaster.max_nodes = $nnodes
108109
multimaster.ignore_tables_without_pk = true
109-
multimaster.twopc_min_timeout = 2000
110-
log_line_prefix = '%t: '
110+
multimaster.twopc_min_timeout = 50000
111+
multimaster.min_2pc_timeout = 50000
112+
log_line_prefix = '%t: '
111113
));
112114

113115
$node->append_conf("pg_hba.conf", qq(

‎contrib/mmts/Dockerfile

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,13 @@
11
FROM kelvich/postgres_cluster
2-
# RUN sysctl -w kernel.core_pattern=core
3-
RUN cd /pg/src/contrib/raftable && make clean && make install
42

53
RUN mkdir /pg/mmts
64
COPY ./ /pg/mmts/
75

8-
RUN export RAFTABLE_PATH=/pg/src/contrib/raftable && \
9-
export USE_PGXS=1 && \
6+
RUN export USE_PGXS=1 && \
107
cd /pg/mmts && make clean && make install
118

129
# pg_regress client assumes such dir exists on server
13-
10+
RUN cp /pg/src/src/test/regress/*.so /pg/install/lib/postgresql/
1411
USER postgres
1512
RUN mkdir /pg/src/src/test/regress/results
1613
ENV PGDATA /pg/data

‎contrib/mmts/Makefile

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,5 @@
11
MODULE_big = multimaster
2-
OBJS = multimaster.o raftable.o arbiter.o bytebuf.o bgwpool.o pglogical_output.o pglogical_proto.o pglogical_receiver.o pglogical_apply.o pglogical_hooks.o pglogical_config.o pglogical_relid_map.o ddd.o bkb.o spill.o
3-
4-
ifndef RAFTABLE_PATH
5-
RAFTABLE_PATH = ../raftable
6-
endif
7-
8-
override CPPFLAGS += -I$(RAFTABLE_PATH) -I$(RAFTABLE_PATH)/raft/include
9-
10-
EXTRA_INSTALL = contrib/raftable contrib/mmts
2+
OBJS = multimaster.o arbiter.o bytebuf.o bgwpool.o pglogical_output.o pglogical_proto.o pglogical_receiver.o pglogical_apply.o pglogical_hooks.o pglogical_config.o pglogical_relid_map.o ddd.o bkb.o spill.o
113

124
EXTENSION = multimaster
135
DATA = multimaster--1.0.sql

‎contrib/mmts/README.md

Lines changed: 23 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,14 @@ Multi-master is an extension and set of patches to a Postegres database, that tu
44
synchronous shared-nothing cluster to provide OLTP scalability and high availability with automatic
55
disaster recovery.
66

7-
8-
97
## Features
108

119
* Cluster-wide transaction isolation
1210
* Synchronous logical replication
1311
* DDL Replication
14-
* Distributed sequences
1512
* Fault tolerance
1613
* Automatic node recovery
1714

18-
19-
2015
## Overview
2116

2217
Multi-master replicates same database to all nodes in cluster and allows writes to each node. Transaction
@@ -65,62 +60,51 @@ After that everything is ready to install postgres along with extensions
6560
git clone https://github.com/postgrespro/postgres_cluster.git
6661
cd postgres_cluster
6762
./configure && make && make -j 4 install
68-
cd ./contrib/raftable && make install
6963
cd ../../contrib/mmts && make install
7064
```
7165

7266
### Docker
7367

74-
Directort contrib/mmts also includes Dockerfile that is capable of building multi-master and starting 3 node cluster.
68+
Directory contrib/mmts also includes docker-compose.yml that is capable of building multi-master and starting 3 node cluster.
7569

7670
```sh
7771
cd contrib/mmts
78-
docker-compose build
7972
docker-compose up
8073
```
8174

8275
### PgPro packages
8376

8477
After things go more stable we will release prebuilt packages for major platforms.
8578

86-
87-
8879
## Configuration
8980

9081
1. Add these required options to the `postgresql.conf` of each instance in the cluster.
91-
9282
```sh
93-
max_prepared_transactions = 200 # should be > 0, because all
94-
# transactions are implicitly two-phase
95-
max_connections = 200
96-
max_worker_processes = 100 # at least (2 * n + p + 1)
97-
# this figure is calculated as:
98-
# 1 raftable worker
99-
# n-1 receiver
100-
# n-1 sender
83+
wal_level = logical # multimaster is build on top of
84+
# logical replication and will not work otherwise
85+
max_connections = 100
86+
max_prepared_transactions = 300 # all transactions are implicitly two-phase, so that's
87+
# a good idea to set this equal to max_connections*N_nodes.
88+
max_wal_senders = 10 # at least the number of nodes
89+
max_replication_slots = 10 # at least the number of nodes
90+
max_worker_processes = 250 # Each node has:
91+
# N_nodes-1 receiver
92+
# N_nodes-1 sender
10193
# 1 mtm-sender
10294
# 1 mtm-receiver
103-
# p workers in the pool
104-
max_parallel_degree = 0
105-
wal_level = logical # multimaster is build on top of
106-
# logical replication and will not work otherwise
107-
max_wal_senders = 10 # at least the number of nodes
108-
wal_sender_timeout = 0
109-
default_transaction_isolation = 'repeatable read'
110-
max_replication_slots = 10 # at least the number of nodes
111-
shared_preload_libraries = 'raftable,multimaster'
112-
multimaster.workers = 10
113-
multimaster.queue_size = 10485760 # 10mb
114-
multimaster.node_id = 1 # the 1-based index of the node in the cluster
115-
multimaster.conn_strings = 'dbname=... host=....0.0.1 port=... raftport=..., ...'
116-
# comma-separated list of connection strings
117-
multimaster.use_raftable = true
118-
multimaster.heartbeat_recv_timeout = 1000
119-
multimaster.heartbeat_send_timeout = 250
120-
multimaster.ignore_tables_without_pk = true
121-
multimaster.twopc_min_timeout = 2000
95+
# Also transactions executed at neighbour nodes can cause spawn of
96+
# background pool worker at our node. At max this will be equal to
97+
# sum of max_connections on neighbour nodes.
98+
99+
100+
101+
shared_preload_libraries = 'multimaster'
102+
multimaster.max_nodes = 3 # cluster size
103+
multimaster.node_id = 1 # the 1-based index of the node in the cluster
104+
multimaster.conn_strings = 'dbname=mydb host=node1.mycluster, ...'
105+
# comma-separated list of connection strings to neighbour nodes.
122106
```
123-
1. Allow replication in `pg_hba.conf`.
107+
2. Allow replication in `pg_hba.conf`.
124108

125109
Read description of all configuration params at [configuration](/contrib/mmts/doc/configuration.md)
126110

@@ -147,16 +131,6 @@ Read description of all management functions at [functions](/contrib/mmts/doc/fu
147131

148132
(Link to test/failure matrix)
149133

150-
### Postgres compatibility
151-
152-
Regression: 141 of 164
153-
Isolation: n/a
154-
155-
To run tests:
156-
* `make -C contrib/mmts check` to run TAP-tests.
157-
* `make -C contrib/mmts xcheck` to run blockade tests. The blockade tests require `docker`, `blockade`, and some other packages installed, see [requirements.txt](tests2/requirements.txt) for the list. You might also want to gain superuser privileges to run these tests successfully.
158-
159-
160134

161135
## Limitations
162136

0 commit comments

Comments
 (0)