Skip to content

Commit f7cf59c

Browse files
knizhnikknizhnik
committed
2 parents edfb332 + af2424d commit f7cf59c

File tree

17 files changed

+116
-89
lines changed

17 files changed

+116
-89
lines changed

‎contrib/pg_dtm/dtmd/include/limits.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,6 @@
66

77
#define BUFFER_SIZE (64 * 1024)
88
#define LISTEN_QUEUE_SIZE 100
9-
#define MAX_STREAMS 128
9+
#define MAX_STREAMS 1024
1010

1111
#endif

‎contrib/pg_dtm/dtmd/include/server.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#define SERVER_H
33

44
#include <stdbool.h>
5+
#include "int.h"
56

67
/*
78
* You should not want to know what is inside those structures.
@@ -95,6 +96,6 @@ bool client_message_finish(client_t client);
9596
*
9697
* Returns 'true' on success, 'false' otherwise.
9798
*/
98-
bool client_message_shortcut(client_t client, long long arg);
99+
bool client_message_shortcut(client_t client, xid_t arg);
99100

100101
#endif

‎contrib/pg_dtm/dtmd/src/clog.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ int clog_read(clog_t clog, xid_t xid) {
133133
bool clog_write(clog_t clog, xid_t xid, int status) {
134134
clogfile_t *file = clog_xid_to_file(clog, xid);
135135
if (!file) {
136-
debug("xid %016llx out of range, creating the file\n", xid);
136+
debug("xid %u out of range, creating the file\n", xid);
137137
clogfile_t newfile;
138138
if (!clogfile_open_by_id(&newfile, clog->datadir, XID_TO_FILEID(xid), true)) {
139139
shout(

‎contrib/pg_dtm/dtmd/src/server.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -325,7 +325,7 @@ bool client_message_finish(client_t client) {
325325
return stream_message_finish(client->stream);
326326
}
327327

328-
bool client_message_shortcut(client_t client, long long arg) {
328+
bool client_message_shortcut(client_t client, xid_t arg) {
329329
if (!stream_message_start(client->stream, client->chan)) {
330330
return false;
331331
}

‎contrib/pg_dtm/libdtm.c

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,9 @@ static DTMConn DtmConnect(char *host, int port)
3939
DTMConn dtm;
4040
int sd;
4141

42-
if (strcmp(host, "localhost") == 0)
42+
if (host == NULL)
4343
{
44+
// use a UNIX socket
4445
struct sockaddr sock;
4546
int len = offsetof(struct sockaddr, sa_data) + snprintf(sock.sa_data, sizeof(sock.sa_data), "%s/p%u", dtm_unix_sock_dir, port);
4647
sock.sa_family = AF_UNIX;
@@ -62,6 +63,7 @@ static DTMConn DtmConnect(char *host, int port)
6263
}
6364
else
6465
{
66+
// use an IP socket
6567
struct addrinfo *addrs = NULL;
6668
struct addrinfo hint;
6769
char portstr[6];
@@ -227,7 +229,9 @@ void DtmGlobalConfig(char *host, int port, char* sock_dir) {
227229
free(dtmhost);
228230
dtmhost = NULL;
229231
}
230-
dtmhost = strdup(host);
232+
if (host) {
233+
dtmhost = strdup(host);
234+
}
231235
dtmport = port;
232236
dtm_unix_sock_dir = sock_dir;
233237
}
@@ -237,14 +241,14 @@ static DTMConn GetConnection()
237241
static DTMConn dtm = NULL;
238242
if (dtm == NULL)
239243
{
240-
if (dtmhost) {
241-
dtm = DtmConnect(dtmhost, dtmport);
242-
if (dtm == NULL)
243-
{
244-
elog(ERROR, "Failed to connect to DTMD %s:%d", dtmhost, dtmport);
244+
dtm = DtmConnect(dtmhost, dtmport);
245+
if (dtm == NULL)
246+
{
247+
if (dtmhost) {
248+
elog(ERROR, "Failed to connect to DTMD at tcp %s:%d", dtmhost, dtmport);
249+
} else {
250+
elog(ERROR, "Failed to connect to DTMD at unix %d", dtmport);
245251
}
246-
} else {
247-
/* elog(ERROR, "DTMD address not specified"); */
248252
}
249253
}
250254
return dtm;

‎contrib/pg_dtm/pg_dtm.c

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -654,9 +654,6 @@ static void DtmInitialize()
654654
dtm->nReservedXids = 0;
655655
dtm->minXid = InvalidTransactionId;
656656
RegisterXactCallback(DtmXactCallback, NULL);
657-
if (DtmBufferSize != 0) {
658-
RegisterBackgroundWorker(&DtmWorker);
659-
}
660657
}
661658
LWLockRelease(AddinShmemInitLock);
662659

@@ -793,7 +790,13 @@ _PG_init(void)
793790
NULL
794791
);
795792

796-
DtmGlobalConfig(DtmHost, DtmPort, Unix_socket_directories);
793+
794+
if (DtmBufferSize != 0) {
795+
DtmGlobalConfig(NULL, DtmPort, Unix_socket_directories);
796+
RegisterBackgroundWorker(&DtmWorker);
797+
} else {
798+
DtmGlobalConfig(DtmHost, DtmPort, Unix_socket_directories);
799+
}
797800

798801
/*
799802
* Install hooks.
@@ -899,9 +902,6 @@ void DtmBackgroundWorker(Datum arg)
899902
params.file = unix_sock_path;
900903
params.buffer_size = DtmBufferSize;
901904

902-
DtmGlobalConfig("localhost", DtmPort, Unix_socket_directories);
903-
904905
ShubInitialize(&shub, &params);
905-
906906
ShubLoop(&shub);
907907
}

‎contrib/pg_dtm/tests/daemons.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ func postgres(bin string, datadir string, port int, nodeid int, wg *sync.WaitGro
9191
bin,
9292
"-D", datadir,
9393
"-p", strconv.Itoa(port),
94+
"-c", "dtm.buffer_size=65536",
9495
"-c", "dtm.host=127.0.0.1",
9596
"-c", "dtm.port=" + strconv.Itoa(5431),
9697
"-c", "autovacuum=off",

‎contrib/pg_dtm/tests/deploy_layouts/cluster.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
---
2-
- hosts: master-workers
2+
- hosts: dtm
33
roles:
44
- role: postgrespro
55
deploy_dtm: true
@@ -10,5 +10,5 @@
1010
pg_port: 15432
1111
deploy_postgres: true
1212
pg_dtm_enable: true
13-
pg_dtm_host: "{{groups['master-workers'][0]}}"
13+
pg_dtm_host: "{{ groups['dtm'][0] }}"
1414

‎contrib/pg_dtm/tests/deploy_layouts/cluster_nodtm.yml

Lines changed: 0 additions & 13 deletions
This file was deleted.

‎contrib/pg_dtm/tests/deploy_layouts/cluster_pg_shard.yml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
---
2+
- hosts: master-workers
3+
roles:
4+
- role: postgrespro
5+
deploy_dtm: true
6+
7+
- hosts: master-workers:workers
8+
roles:
9+
- role: postgrespro
10+
pg_port: 15432
11+
deploy_postgres: true
12+
deploy_pg_shard: true
13+
pg_dtm_enable: true
14+
pg_dtm_host: "{{ groups['master-workers'][0] }}"
15+

‎contrib/pg_dtm/tests/deploy_layouts/cluster_pg_shard_nodtm.yml

Lines changed: 0 additions & 40 deletions
This file was deleted.

‎contrib/pg_dtm/tests/farms/localhost

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[master-workers]
2+
localhost

‎contrib/pg_dtm/tests/farms/mephi

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
1-
[client]
1+
[clients]
22
br.theor.mephi.ru
3+
blade3 ansible_ssh_host=85.143.113.31 ansible_ssh_port=4203
34

45
[workers]
5-
blade3 ansible_ssh_host=85.143.113.31 ansible_ssh_port=4203
66
blade4 ansible_ssh_host=85.143.113.31 ansible_ssh_port=4204
77
blade5 ansible_ssh_host=85.143.113.31 ansible_ssh_port=4205
88
blade6 ansible_ssh_host=85.143.113.31 ansible_ssh_port=4206
99
blade7 ansible_ssh_host=85.143.113.31 ansible_ssh_port=4207
1010
blade8 ansible_ssh_host=85.143.113.31 ansible_ssh_port=4208
1111

12-
[master-workers]
12+
[dtm]
1313
blade8 ansible_ssh_host=85.143.113.31 ansible_ssh_port=4208
1414

1515

‎contrib/pg_dtm/tests/farms/sai

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
1+
[clients]
2+
158.250.29.4 ansible_ssh_user=s.kelvich
3+
158.250.29.5 ansible_ssh_user=s.kelvich
4+
15
[workers]
2-
astro4 ansible_ssh_host=158.250.29.4 ansible_ssh_user=s.kelvich
3-
astro5 ansible_ssh_host=158.250.29.5 ansible_ssh_user=s.kelvich
4-
astro6 ansible_ssh_host=158.250.29.6 ansible_ssh_user=s.kelvich
5-
astro8 ansible_ssh_host=158.250.29.8 ansible_ssh_user=s.kelvich
6-
astro9 ansible_ssh_host=158.250.29.9 ansible_ssh_user=s.kelvich
6+
158.250.29.6 ansible_ssh_user=s.kelvich
7+
158.250.29.8 ansible_ssh_user=s.kelvich
8+
158.250.29.9 ansible_ssh_user=s.kelvich
9+
158.250.29.10 ansible_ssh_user=s.kelvich
710

8-
[master-workers]
9-
astro10 ansible_ssh_host=158.250.29.10 ansible_ssh_user=s.kelvich
11+
[dtm]
12+
158.250.29.10 ansible_ssh_user=s.kelvich

‎contrib/pg_dtm/tests/transfers-remote.sh

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,15 @@
11
#!/bin/sh
22

3+
FARM="sai"
4+
WORKERS=3
5+
TRANS_SRV="s.kelvich@158.250.29.10"
6+
7+
38
if [ ! -f ./transfers.linux ]; then
4-
GOOS=linux GOARCH=amd64 go build -o ./transfers ./transfers.go
9+
GOOS=linux GOARCH=amd64 go build -o ./transfers.linux ./transfers.go
510
fi
611

7-
ansible master-workers -i farms/mephi -m copy -a "src=transfers dest=~/transfers mode=a+x"
12+
ansible master-workers -i farms/$FARM -m copy -a "src=transfers.linux dest=~/transfers mode=a+x"
813

914

1015

@@ -18,3 +23,17 @@ ssh -p4207 br.theor.mephi.ru "./transfers -d 'host=blade8 dbname=postgres user=s
1823

1924

2025

26+
ssh s.kelvich@158.250.29.10 "./transfers -d 'host=158.250.29.9 port=15432 dbname=postgres' -d 'host=158.250.29.8 port=15432 dbname=postgres' -d 'host=158.250.29.4 port=15432 dbname=postgres' -d 'host=158.250.29.5 port=15432 dbname=postgres' -d 'host=158.250.29.6 port=15432 dbname=postgres' -g -w 64"
27+
28+
29+
ssh s.kelvich@158.250.29.10 "./transfers -d 'host=158.250.29.9 user=s.kelvich port=15432 dbname=postgres' -g -w 128"
30+
31+
ssh s.kelvich@158.250.29.10 "./transfers -d 'host=158.250.29.9 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.8 user=s.kelvich port=15432 dbname=postgres' -g -w 128"
32+
33+
ssh s.kelvich@158.250.29.10 "./transfers -d 'host=158.250.29.9 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.8 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.6 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.5 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.4 user=s.kelvich port=15432 dbname=postgres' -g -w 128"
34+
35+
36+
37+
38+
./transfers -d 'host=127.0.0.1 port=5432 dbname=postgres' -d 'host=127.0.0.1 port=5433 dbname=postgres' -g -w 128
39+

‎contrib/pg_dtm/tests/transfers.go

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ var cfg struct {
3232

3333
Verbose bool
3434
UseDtm bool
35+
InitOnly bool
36+
SkipInit bool
37+
3538
Isolation string // "repeatable read" or "read committed"
3639

3740
Accounts struct {
@@ -108,6 +111,8 @@ func init() {
108111
flag.BoolVar(&cfg.Writers.AllowLocal, "l", false, "Allow local updates")
109112
flag.BoolVar(&cfg.Writers.PrivateRows, "p", false, "Private rows (avoid waits/aborts caused by concurrent updates of the same rows)")
110113
flag.BoolVar(&cfg.Writers.UseCursors, "c", false, "Use cursors for updates")
114+
flag.BoolVar(&cfg.InitOnly, "f", false, "Only feed databses with data")
115+
flag.BoolVar(&cfg.SkipInit, "s", false, "Skip init phase")
111116
flag.Parse()
112117

113118
if len(cfg.ConnStrs) == 0 {
@@ -141,8 +146,15 @@ func init() {
141146

142147
func main() {
143148
start := time.Now()
144-
prepare(cfg.ConnStrs)
145-
fmt.Printf("database prepared in %0.2f seconds\n", time.Since(start).Seconds())
149+
150+
if (!cfg.SkipInit){
151+
prepare(cfg.ConnStrs)
152+
fmt.Printf("database prepared in %0.2f seconds\n", time.Since(start).Seconds())
153+
}
154+
155+
if (cfg.InitOnly) {
156+
return
157+
}
146158

147159
var writerWg sync.WaitGroup
148160
var readerWg sync.WaitGroup

‎contrib/pg_dtm/tests/transfers.yml

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
---
2+
3+
- hosts: masters
4+
gather_facts: no
5+
tasks:
6+
- name: copy transfers binary
7+
copy: src=transfers.linux dest=~/transfers mode=a+x
8+
9+
- hosts: 158.250.29.4
10+
gather_facts: no
11+
tasks:
12+
- name: setup the databases
13+
shell: "./transfers -d 'host=158.250.29.10 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.9 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.8 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.5 user=s.kelvich port=15432 dbname=postgres' -f -g -w 200 -r 0"
14+
register: transfers_result
15+
- debug: var=transfers_result
16+
17+
- hosts: masters
18+
gather_facts: no
19+
tasks:
20+
- name: run transfers
21+
shell: "./transfers -d 'host=158.250.29.10 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.9 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.8 user=s.kelvich port=15432 dbname=postgres' -d 'host=158.250.29.5 user=s.kelvich port=15432 dbname=postgres' -s -g -w 200 -r 0"
22+
register: transfers_result
23+
- debug: var=transfers_result

0 commit comments

Comments
 (0)