Skip to content

Commit fc0c4e5

Browse files
kvapkvap
committed
Merge branch 'master' into contain
2 parents cd303d8 + 29c7901 commit fc0c4e5

File tree

469 files changed

+19866
-5694
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

469 files changed

+19866
-5694
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ win32ver.rc
2727
*.exe
2828
lib*dll.def
2929
lib*.pc
30+
.DS_Store
3031

3132
# Local excludes in root directory
3233
/GNUmakefile

‎README.md

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Postgres_cluster
2+
3+
Various experiments with PostgreSQL clustering perfomed at PostgresPro.
4+
5+
This is mirror of postgres repo with several changes to the core and few extra extensions.
6+
7+
## Core changes:
8+
9+
* Transaction manager interface (eXtensible Transaction Manager, xtm). Generic interface to plug distributed transaction engines. More info at [[https://wiki.postgresql.org/wiki/DTM]] and [[http://www.postgresql.org/message-id/flat/F2766B97-555D-424F-B29F-E0CA0F6D1D74@postgrespro.ru]].
10+
* Distributed deadlock detection API.
11+
* Fast 2pc patch. More info at [[http://www.postgresql.org/message-id/flat/74355FCF-AADC-4E51-850B-47AF59E0B215@postgrespro.ru]]
12+
13+
## New extensions:
14+
15+
* pg_dtm. Transaction management by interaction with standalone coordinator (Arbiter or dtmd). [[https://wiki.postgresql.org/wiki/DTM#DTM_approach]]
16+
* pg_tsdtm. Coordinator-less transaction management by tracking commit timestamps.
17+
* multimaster. Synchronous multi-master replication based on logical_decoding and pg_dtm.
18+
19+
20+
## Changed extension:
21+
22+
* postgres_fdw. Added support of pg_dtm.
23+
24+
## Deploying
25+
26+
For deploy and test postgres over a cluster we use ansible. In each extension directory one can find test subdirectory where we are storing tests and deploy scripts.
27+
28+
29+
### Running tests on local cluster
30+
31+
To use it one need ansible hosts file with following groups:
32+
33+
farms/cluster.example:
34+
```
35+
[clients] # benchmark will start simultaneously on that nodes
36+
server0.example.com
37+
[nodes] # all that nodes will run postgres, dtmd/master will be deployed to first
38+
server1.example.com
39+
server2.example.com
40+
server3.example.com
41+
```
42+
43+
After you have proper hosts file you can deploy all stuff to servers:
44+
45+
```shell
46+
# cd pg_dtm/tests
47+
# ansible-playbook -i farms/sai deploy_layouts/cluster.yml
48+
```
49+
50+
To perform dtmbench run:
51+
52+
```shell
53+
# ansible-playbook -i farms/sai perf.yml -e nnodes=3 -e nconns=100
54+
```
55+
56+
here nnodes is number of nudes that will be used for that test, nconns is the
57+
number of connections to the backend.
58+
59+
60+
61+
## Running tests on Amazon ec2
62+
63+
64+
In the case of amazon cloud there is no need in specific hosts file. Instead of it
65+
we use script farms/ec2.py to get current instances running on you account. To use
66+
that script you need to specify you account key and access_key in ~/.boto.cfg (or in
67+
any other place that described at http://boto.cloudhackers.com/en/latest/boto_config_tut.html)
68+
69+
To create VMs in cloud run:
70+
```shell
71+
# ansible-playbook -i farms/ec2.py deploy_layouts/ec2.yml
72+
```
73+
After that you should wait few minutes to have info about that instances in Amazon API. After
74+
that you can deploy postgres as usual:
75+
```shell
76+
# ansible-playbook -i farms/ec2.py deploy_layouts/cluster-ec2.yml
77+
```
78+
And to run a benchmark:
79+
```shell
80+
# ansible-playbook -i farms/sai perf-ec2.yml -e nnodes=3 -e nconns=100
81+
```

‎config/Makefile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,11 @@ include $(top_builddir)/src/Makefile.global
77

88
install: all installdirs
99
$(INSTALL_SCRIPT) $(srcdir)/install-sh '$(DESTDIR)$(pgxsdir)/config/install-sh'
10+
$(INSTALL_SCRIPT) $(srcdir)/missing '$(DESTDIR)$(pgxsdir)/config/missing'
1011

1112
installdirs:
1213
$(MKDIR_P) '$(DESTDIR)$(pgxsdir)/config'
1314

1415
uninstall:
1516
rm -f '$(DESTDIR)$(pgxsdir)/config/install-sh'
17+
rm -f '$(DESTDIR)$(pgxsdir)/config/missing'

‎config/programs.m4

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ else
7777
echo '%%' > conftest.l
7878
if $pgac_candidate -t conftest.l 2>/dev/null | grep FLEX_SCANNER >/dev/null 2>&1; then
7979
pgac_flex_version=`$pgac_candidate --version 2>/dev/null`
80-
if echo "$pgac_flex_version" | sed ['s/[.a-z]/ /g'] | $AWK '{ if ([$]1 = 2 && [$]2 = 5 && [$]3 >= 31) exit 0; else exit 1;}'
80+
if echo "$pgac_flex_version" | sed ['s/[.a-z]/ /g'] | $AWK '{ if ([$]1 = 2 && ([$]2 > 5 || ([$]2 = 5 && [$]3 >= 31))) exit 0; else exit 1;}'
8181
then
8282
pgac_cv_path_flex=$pgac_candidate
8383
break 2

‎configure

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7272,7 +7272,7 @@ else
72727272
echo '%%' > conftest.l
72737273
if $pgac_candidate -t conftest.l 2>/dev/null | grep FLEX_SCANNER >/dev/null 2>&1; then
72747274
pgac_flex_version=`$pgac_candidate --version 2>/dev/null`
7275-
if echo "$pgac_flex_version" | sed 's/[.a-z]/ /g' | $AWK '{ if ($1 = 2 && $2 = 5 && $3 >= 31) exit 0; else exit 1;}'
7275+
if echo "$pgac_flex_version" | sed 's/[.a-z]/ /g' | $AWK '{ if ($1 = 2 && ($2 > 5 || ($2 = 5 && $3 >= 31))) exit 0; else exit 1;}'
72767276
then
72777277
pgac_cv_path_flex=$pgac_candidate
72787278
break 2

‎contrib/file_fdw/file_fdw.c

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,8 @@ static ForeignScan *fileGetForeignPlan(PlannerInfo *root,
121121
Oid foreigntableid,
122122
ForeignPath *best_path,
123123
List *tlist,
124-
List *scan_clauses);
124+
List *scan_clauses,
125+
Plan *outer_plan);
125126
static void fileExplainForeignScan(ForeignScanState *node, ExplainState *es);
126127
static void fileBeginForeignScan(ForeignScanState *node, int eflags);
127128
static TupleTableSlot *fileIterateForeignScan(ForeignScanState *node);
@@ -525,6 +526,7 @@ fileGetForeignPaths(PlannerInfo *root,
525526
total_cost,
526527
NIL, /* no pathkeys */
527528
NULL, /* no outer rel either */
529+
NULL, /* no extra plan */
528530
coptions));
529531

530532
/*
@@ -544,7 +546,8 @@ fileGetForeignPlan(PlannerInfo *root,
544546
Oid foreigntableid,
545547
ForeignPath *best_path,
546548
List *tlist,
547-
List *scan_clauses)
549+
List *scan_clauses,
550+
Plan *outer_plan)
548551
{
549552
Index scan_relid = baserel->relid;
550553

@@ -564,7 +567,8 @@ fileGetForeignPlan(PlannerInfo *root,
564567
NIL, /* no expressions to evaluate */
565568
best_path->fdw_private,
566569
NIL, /* no custom tlist */
567-
NIL /* no remote quals */ );
570+
NIL, /* no remote quals */
571+
outer_plan);
568572
}
569573

570574
/*

‎contrib/hstore/hstore.h

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -76,11 +76,11 @@ typedef struct
7676
#define STRPTR(x) ( (char*)(ARRPTR(x) + HS_COUNT((HStore*)(x)) * 2) )
7777

7878
/* note multiple/non evaluations */
79-
#define HS_KEY(arr_,str_,i_) ((str_) + HSE_OFF((arr_)[2*(i_)]))
80-
#define HS_VAL(arr_,str_,i_) ((str_) + HSE_OFF((arr_)[2*(i_)+1]))
81-
#define HS_KEYLEN(arr_,i_) (HSE_LEN((arr_)[2*(i_)]))
82-
#define HS_VALLEN(arr_,i_) (HSE_LEN((arr_)[2*(i_)+1]))
83-
#define HS_VALISNULL(arr_,i_) (HSE_ISNULL((arr_)[2*(i_)+1]))
79+
#define HSTORE_KEY(arr_,str_,i_) ((str_) + HSE_OFF((arr_)[2*(i_)]))
80+
#define HSTORE_VAL(arr_,str_,i_) ((str_) + HSE_OFF((arr_)[2*(i_)+1]))
81+
#define HSTORE_KEYLEN(arr_,i_) (HSE_LEN((arr_)[2*(i_)]))
82+
#define HSTORE_VALLEN(arr_,i_) (HSE_LEN((arr_)[2*(i_)+1]))
83+
#define HSTORE_VALISNULL(arr_,i_) (HSE_ISNULL((arr_)[2*(i_)+1]))
8484

8585
/*
8686
* currently, these following macros are the _only_ places that rely

‎contrib/hstore/hstore_compat.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ hstoreValidNewFormat(HStore *hs)
149149

150150
for (i = 1; i < count; ++i)
151151
{
152-
if (HS_KEYLEN(entries, i) < HS_KEYLEN(entries, i - 1))
152+
if (HSTORE_KEYLEN(entries, i) < HSTORE_KEYLEN(entries, i - 1))
153153
return 0;
154154
if (HSE_ISNULL(entries[2 * i]))
155155
return 0;

‎contrib/hstore/hstore_gin.c

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,14 +59,16 @@ gin_extract_hstore(PG_FUNCTION_ARGS)
5959
{
6060
text *item;
6161

62-
item = makeitem(HS_KEY(hsent, ptr, i), HS_KEYLEN(hsent, i),
62+
item = makeitem(HSTORE_KEY(hsent, ptr, i),
63+
HSTORE_KEYLEN(hsent, i),
6364
KEYFLAG);
6465
entries[2 * i] = PointerGetDatum(item);
6566

66-
if (HS_VALISNULL(hsent, i))
67+
if (HSTORE_VALISNULL(hsent, i))
6768
item = makeitem(NULL, 0, NULLFLAG);
6869
else
69-
item = makeitem(HS_VAL(hsent, ptr, i), HS_VALLEN(hsent, i),
70+
item = makeitem(HSTORE_VAL(hsent, ptr, i),
71+
HSTORE_VALLEN(hsent, i),
7072
VALFLAG);
7173
entries[2 * i + 1] = PointerGetDatum(item);
7274
}

‎contrib/hstore/hstore_gist.c

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -129,11 +129,13 @@ ghstore_compress(PG_FUNCTION_ARGS)
129129
{
130130
int h;
131131

132-
h = crc32_sz((char *) HS_KEY(hsent, ptr, i), HS_KEYLEN(hsent, i));
132+
h = crc32_sz((char *) HSTORE_KEY(hsent, ptr, i),
133+
HSTORE_KEYLEN(hsent, i));
133134
HASH(GETSIGN(res), h);
134-
if (!HS_VALISNULL(hsent, i))
135+
if (!HSTORE_VALISNULL(hsent, i))
135136
{
136-
h = crc32_sz((char *) HS_VAL(hsent, ptr, i), HS_VALLEN(hsent, i));
137+
h = crc32_sz((char *) HSTORE_VAL(hsent, ptr, i),
138+
HSTORE_VALLEN(hsent, i));
137139
HASH(GETSIGN(res), h);
138140
}
139141
}
@@ -524,13 +526,15 @@ ghstore_consistent(PG_FUNCTION_ARGS)
524526

525527
for (i = 0; res && i < count; ++i)
526528
{
527-
int crc = crc32_sz((char *) HS_KEY(qe, qv, i), HS_KEYLEN(qe, i));
529+
int crc = crc32_sz((char *) HSTORE_KEY(qe, qv, i),
530+
HSTORE_KEYLEN(qe, i));
528531

529532
if (GETBIT(sign, HASHVAL(crc)))
530533
{
531-
if (!HS_VALISNULL(qe, i))
534+
if (!HSTORE_VALISNULL(qe, i))
532535
{
533-
crc = crc32_sz((char *) HS_VAL(qe, qv, i), HS_VALLEN(qe, i));
536+
crc = crc32_sz((char *) HSTORE_VAL(qe, qv, i),
537+
HSTORE_VALLEN(qe, i));
534538
if (!GETBIT(sign, HASHVAL(crc)))
535539
res = false;
536540
}

0 commit comments

Comments
 (0)