This repo implements distributed transaction manager using Snapshot Sharing mechanism. General concepts and alternative approaches described in postgres wiki https://wiki.postgresql.org/wiki/DTM.

Backend-DTM protocol description can be found in [dtmd/README](dtmd/README).

* Patch postgres using xtm.patch. After that build and install postgres in usual way.

cd ~/code/postgres

patch -p1 < ~/code/pg_dtm/xtm.patch

* Install pg_dtm extension.

export PATH=/path/to/pgsql/bin/:$PATH

cd ~/code/pg_dtm

make && make install

* Run dtmd.

cd ~/code/pg_dtm/dtmd

make

mkdir /tmp/clog

./bin/dtmd &

* To run something meaningful you need at leat two postgres instances. Also pg_dtm requires presense in ```shared_preload_libraries```.

initdb -D ./install/data1

initdb -D ./install/data2

echo "port = 5433" >> ./install/data2/postgresql.conf

echo "shared_preload_libraries = 'pg_dtm'" >> ./install/data1/postgresql.conf

echo "shared_preload_libraries = 'pg_dtm'" >> ./install/data2/postgresql.conf

pg_ctl -D ./install/data1 -l ./install/data1/log start

pg_ctl -D ./install/data2 -l ./install/data2/log start

For a cluster-wide deploy we use ansible, more details in tests/deploy_layouts. (Ansible instructions will be later)

Now cluster is running and you can use global tx between two nodes. Let's connect to postgres instances at different ports:

create extension pg_dtm; -- node1

create table accounts(user_id int, amount int); -- node1

insert into accounts (select 2*generate_series(1,100)-1, 0); -- node1, odd user_id's

create extension pg_dtm; -- node2

create table accounts(user_id int, amount int); -- node2

insert into accounts (select 2*generate_series(1,100), 0); -- node2, even user_id's

select dtm_begin_transaction(); -- node1, returns global xid, e.g. 42

select dtm_join_transaction(42); -- node2, join global tx

begin; -- node1

begin; -- node2

update accounts set amount=amount-100 where user_id=1; -- node1, transfer money from user#1

update accounts set amount=amount+100 where user_id=2; -- node2, to user#2

commit; -- node1, blocks until second commit happend

commit; -- node2

To ensure consistency we use simple bank test: perform a lot of simultaneous transfers between accounts on different servers, while constantly checking total amount of money on all accounts. This test can be found in tests/perf.

> go run ./tests/perf/*

-C value

Connection string (repeat for multiple connections)

-a int

The number of bank accounts (default 100000)

-b string

Backend to use. Possible optinos: transfers, fdw, pgshard, readers. (default "transfers")

-g Use DTM to keep global consistency

-i Init database

-l Use 'repeatable read' isolation level instead of 'read committed'

-n int

The number updates each writer (reader in case of Reades backend) performs (default 10000)

-p Use parallel execs

-r int

The number of readers (default 1)

-s int

StartID. Script will update rows starting from this value

-v Show progress and other stuff for mortals

-w int

The number of writers (default 8)

go run ./tests/perf/*.go \

-C "dbname=postgres port=5432" \

-C "dbname=postgres port=5433" \

-g -i

go run ./tests/perf/*.go \

-C "dbname=postgres port=5432" \

-C "dbname=postgres port=5433" \

-g

We also provide a patch, that enables support of global transactions with postres_fdw. After patching and installing postres_fdw it is possible to run same test via fdw usig key ```-b fdw```.

pg_shard.use_dtm_transactions = 1

bin/*

obj/*

===

xtm

===

Distributed transaction management tools for PostgreSQL.

--------------------

Communication scheme

--------------------

┏━━━━━━━━━┓

┌────────┨ Backend ┠──────────┐

│ ┗━━━━━━━━━┛ │

┏━━━━┷━━━━┓ ┏━━━━━━━━━┓ ┏━━━━━━┷━━━━━━┓

┃ Arbiter ┠───┨ Backend ┠───┨ Coordinator ┃

┗━━━━┯━━━━┛ ┗━━━━━━━━━┛ ┗━━━━━━┯━━━━━━┛

│ ┏━━━━━━━━━┓ │

└──┬─────┨ Backend ┠───────┬──┘

┆ ┗━━━━━━━━━┛ ┆

libdtm + libsockhub libpq + xtm procs

-----------------------

Coordinator-Backend API

-----------------------

This API includes a set of postgres procedures that

the coordinator can call with "select" statement.

FIXME: actualize the API

------------------------

Backend-Arbiter Protocol

------------------------

The underlying protocol (libsockhub) also transmits the message length, so

there is no need in 'argc'. Every command or reply is a series of int64

numbers.

The format of all commands:

[cmd, argv[0], argv[1], ...]

'cmd' is a command.

'argv[i]' are the arguments.

The commands:

'r': reserve(minxid, minsize)

Claims a sequence ≥ minsize of xids ≥ minxid for local usage. This will

prevent the arbiter from using those values for global transactions.

The arbiter replies with:

[RES_OK, min, max] if reserved a range [min, max]

[RES_FAILED] on failure

'b': begin(size)

Starts a global transaction and assign a 'xid' to it. 'size' is used

for vote results calculation. The arbiter also creates and returns the

snapshot.

The arbiter replies with:

[RES_OK, xid, *snapshot] if transaction started successfully

[RES_FAILED] on failure

See the 'snapshot' command description for the snapshot format.

's': status(xid, wait)

Asks the arbiter about the status of the global transaction identified

by the given 'xid'.

If 'wait' is 1, the arbiter will not reply until it considers the

transaction finished (all nodes voted, or one dead).

The arbiter replies with:

[RES_TRANSACTION_UNKNOWN] if not started

[RES_TRANSACTION_COMMITTED] if committed

[RES_TRANSACTION_ABORTED] if aborted

[RES_TRANSACTION_INPROGRESS] if in progress

[RES_FAILED] if failed

'y': for(xid, wait)

Tells the arbiter that this node votes for commit of the global

transaction identified by the given 'xid'.

The reply and 'wait' logic is the same as for the 'status' command.

'n': against(xid, wait)

Tells the arbiter that this node votes againts commit of the global

transaction identified by the given 'xid'.

The reply and 'wait' logic is the same as for the 'status' command.

'h': snapshot(xid)

Tells the arbiter to generate a snapshot for the global transaction

identified by the given 'xid'. The arbiter will create a snapshot for

every participant, so when each of them asks for the snapshot it will

reply with the same snapshot. The arbiter generates a fresh version if

the same client asks for a snapshot again for the same transaction.

Joins the global transaction identified by the given 'xid', if not

joined already.

The arbiter replies with [RES_OK, gxmin, xmin, xmax, xcnt, xip[0], xip[1]...],

where 'gxmin' is the smallest xmin among all available snapshots.

In case of a failure, the arbiter replies with [RES_FAILED].

#ifndef LIMITS_H

#define LIMITS_H

#define MAX_TRANSACTIONS 4096

#define BUFFER_SIZE (256 * 1024)

#define LISTEN_QUEUE_SIZE 100

#define MAX_STREAMS 4096

CC = gcc

CFLAGS = -c -I. -I.. -Wall -O3 -g -fPIC

LD = $(CC)

LDFLAGS = -g

AR = ar

ARFLAGS = -cru

all: sockhub library tests

library: libsockhub.a

tests: test-client test-async-client test-server

sockhub.o: ../sockhub.c ../sockhub.h

$(CC) $(CFLAGS) ../sockhub.c

sockhup_main.o: sockhub_main.c sockhub.h

$(CC) $(CFLAGS) sockhub_main.c

libsockhub.a: sockhub.o

rm -f libsockhub.a

$(AR) $(ARFLAGS) libsockhub.a sockhub.o

sockhub: sockhub_main.o libsockhub.a

$(LD) $(LDFLAGS) -o sockhub sockhub_main.o libsockhub.a

test-client.o: test-client.c ../sockhub.h

$(CC) $(CFLAGS) test-client.c

test-client: test-client.o libsockhub.a

$(LD) $(LDFLAGS) -o test-client test-client.o libsockhub.a

test-async-client.o: test-async-client.c ../sockhub.h

$(CC) $(CFLAGS) test-async-client.c

test-async-client: test-async-client.o libsockhub.a

$(LD) $(LDFLAGS) -o test-async-client test-async-client.o libsockhub.a

test-server.o: test-server.c ../sockhub.h

$(CC) $(CFLAGS) test-server.c

test-server: test-server.o libsockhub.a

$(LD) $(LDFLAGS) -o test-server test-server.o libsockhub.a

clean:

rm -f *.o *.a

tgz: clean

cd .. ; tar cvzf sockhub.tgz sockhub

#include <stdio.h>

#include <stdlib.h>

#include "sockhub.h"

int main(int argc, char* argv[])

{

int i;

Shub shub;

ShubParams params;

ShubInitParams(&params);

for (i = 1; i < argc-1; i++) {

if (argv[i][0] == '-') {

switch (argv[i][1]) {

case 'h':

params.host = argv[++i];

continue;

case 'p':

params.port = atoi(argv[++i]);

continue;

case 'f':

params.file = argv[++i];

continue;

case 'd':

params.delay = atoi(argv[++i]);

continue;

case 'b':

params.buffer_size = atoi(argv[++i]);

continue;

case 'r':

params.max_attempts = atoi(argv[++i]);

continue;

}

}

Usage:

fprintf(stderr, "sockhub: combine several local unix socket connections into one inet socket\n"

"Options:\n"

"\t-h HOST\tremote host name\n"

"\t-p PORT\tremote port\n"

"\t-f FILE\tunix socket file name\n"

"\t-d DELAY\tdelay for waiting income requests (milliseconds)\n"

"\t-b SIZE\tbuffer size\n"

"\t-q SIZE\tlisten queue size\n"

"\t-r N\tmaximun connect attempts\n"

);

return 1;

}

if (params.host == NULL || params.file == NULL || i != argc) {

goto Usage;

}

ShubInitialize(&shub, &params);

ShubLoop(&shub);

return 0;

}

n_clients=8

n_iters=10000

pkill -9 sockub

pkill -9 test-async-client

./sockhub -h $1 -p 5001 -f /tmp/p5002 &

for ((i=0;i<n_clients;i++))

./test-async-client -h localhost -p 5002 -i $n_iters &

n_clients=10

n_iters=100000

for ((i=0;i<n_clients;i++))

./test-client $1 5001 $n_iters &