Skip to content

Commit 3d88652

Browse files
vbwagnervbwagner
committed
Merge commit 'c45a95ab354f03bec617bd62a818d5eadcc06fc0' into PGPROEE_pathman_148
2 parents 7f47619 + c45a95a commit 3d88652

File tree

12 files changed

+303
-300
lines changed

12 files changed

+303
-300
lines changed

‎contrib/pg_pathman/.github/ISSUE_TEMPLATE.md

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<!--
2+
### IMPORTANT: make sure that you:
3+
4+
- are familiar with [known limitations](https://github.com/postgrespro/pg_pathman/wiki/Known-limitations).
5+
- checked the [faq](https://github.com/postgrespro/pg_pathman/wiki/What-the-FAQ) for common problems.
6+
-->
7+
8+
9+
### Problem description
10+
11+
Explain your problem here (it's always better to provide reproduction steps) ...
12+
13+
14+
15+
### Environment
16+
17+
<!-- Put the result of (SELECT * FROM pg_extension) below -->
18+
19+
<!-- Put the result of (SELECT version()) below -->
20+
21+
<!-- For Postgres Pro: put the result of (SELECT pgpro_version()) below -->
22+
23+
<!-- For Postgres Pro: put the result of (SELECT pgpro_edition()) below -->
24+
25+
<!-- For pg_pathman 1.4: put the result of (SELECT get_pathman_lib_version()) below -->
26+
27+
<!-- For pg_pathman 1.5: put the result of (SELECT pathman_version()) below -->
28+

‎contrib/pg_pathman/expected/pathman_bgw.out

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ DECLARE
187187
i int4 := 0; -- protect from endless loop
188188
BEGIN
189189
LOOP
190+
-- get total number of processed rows
190191
SELECT processed
191192
FROM pathman_concurrent_part_tasks
192193
WHERE relid = 'test_bgw.conc_part'::regclass
@@ -200,9 +201,12 @@ BEGIN
200201

201202
ASSERT rows IS NOT NULL;
202203

203-
-- rows should increase!
204-
IF rows_old <= rows THEN
204+
IF rows_old = rows THEN
205205
i = i + 1;
206+
ELSIF rows < rows_old THEN
207+
RAISE EXCEPTION 'rows is decreasing: new %, old %', rows, rows_old;
208+
ELSIF rows > 500 THEN
209+
RAISE EXCEPTION 'processed % rows', rows;
206210
END IF;
207211
ELSE
208212
EXIT; -- exit loop

‎contrib/pg_pathman/expected/pathman_utility_stmt.out

Lines changed: 24 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -24,70 +24,6 @@ VACUUM FULL copy_stmt_hooking.test_1;
2424
VACUUM FULL copy_stmt_hooking.test_2;
2525
VACUUM FULL copy_stmt_hooking.test_3;
2626
VACUUM FULL copy_stmt_hooking.test_4;
27-
/* COPY TO */
28-
COPY copy_stmt_hooking.test TO stdout;
29-
1 comment \N \N
30-
2 comment \N \N
31-
3 comment \N \N
32-
4 comment \N \N
33-
5 comment \N \N
34-
6 comment \N \N
35-
7 comment \N \N
36-
8 comment \N \N
37-
9 comment \N \N
38-
10 comment \N \N
39-
11 comment \N \N
40-
12 comment \N \N
41-
13 comment \N \N
42-
14 comment \N \N
43-
15 comment \N \N
44-
16 comment \N \N
45-
17 comment \N \N
46-
18 comment \N \N
47-
19 comment \N \N
48-
20 comment \N \N
49-
\copy copy_stmt_hooking.test to stdout (format csv)
50-
1,comment,,
51-
2,comment,,
52-
3,comment,,
53-
4,comment,,
54-
5,comment,,
55-
6,comment,,
56-
7,comment,,
57-
8,comment,,
58-
9,comment,,
59-
10,comment,,
60-
11,comment,,
61-
12,comment,,
62-
13,comment,,
63-
14,comment,,
64-
15,comment,,
65-
16,comment,,
66-
17,comment,,
67-
18,comment,,
68-
19,comment,,
69-
20,comment,,
70-
\copy copy_stmt_hooking.test(comment) to stdout
71-
comment
72-
comment
73-
comment
74-
comment
75-
comment
76-
comment
77-
comment
78-
comment
79-
comment
80-
comment
81-
comment
82-
comment
83-
comment
84-
comment
85-
comment
86-
comment
87-
comment
88-
comment
89-
comment
90-
comment
9127
/* DELETE ROWS, COPY FROM */
9228
DELETE FROM copy_stmt_hooking.test;
9329
COPY copy_stmt_hooking.test FROM stdin;
@@ -113,32 +49,30 @@ VACUUM FULL copy_stmt_hooking.test_1;
11349
VACUUM FULL copy_stmt_hooking.test_2;
11450
VACUUM FULL copy_stmt_hooking.test_3;
11551
VACUUM FULL copy_stmt_hooking.test_4;
116-
/* COPY FROM (specified columns) */
117-
COPY copy_stmt_hooking.test (val) TO stdout;
118-
1
119-
6
120-
7
121-
11
122-
16
123-
COPY copy_stmt_hooking.test (val, comment) TO stdout;
124-
1 test_1
125-
6 test_2
126-
7 test_2
127-
11 test_3
128-
16 test_4
129-
COPY copy_stmt_hooking.test (c3, val, comment) TO stdout;
130-
0 1 test_1
131-
0 6 test_2
132-
0 7 test_2
133-
0 11 test_3
134-
0 16 test_4
135-
COPY copy_stmt_hooking.test (val, comment, c3, c4) TO stdout;
52+
/* COPY TO */
53+
COPY copy_stmt_hooking.test TO stdout; /* not ok */
54+
WARNING: COPY TO will only select rows from parent table "test"
55+
COPY copy_stmt_hooking.test (val) TO stdout; /* not ok */
56+
WARNING: COPY TO will only select rows from parent table "test"
57+
COPY (SELECT * FROM copy_stmt_hooking.test) TO stdout;
58+
1 test_1 0 0
59+
6 test_2 0 0
60+
7 test_2 0 0
61+
11 test_3 0 0
62+
16 test_4 0 0
63+
COPY (SELECT * FROM copy_stmt_hooking.test) TO stdout (FORMAT CSV);
64+
1,test_1,0,0
65+
6,test_2,0,0
66+
7,test_2,0,0
67+
11,test_3,0,0
68+
16,test_4,0,0
69+
\copy (SELECT * FROM copy_stmt_hooking.test) TO stdout
13670
1 test_1 0 0
13771
6 test_2 0 0
13872
7 test_2 0 0
13973
11 test_3 0 0
14074
16 test_4 0 0
141-
/* COPY TO (partition does not exist, NOT allowed to create partitions) */
75+
/* COPY FROM (partition does not exist, NOT allowed to create partitions) */
14276
SET pg_pathman.enable_auto_partition = OFF;
14377
COPY copy_stmt_hooking.test FROM stdin;
14478
ERROR: no suitable partition for key '21'
@@ -147,7 +81,7 @@ SELECT * FROM copy_stmt_hooking.test WHERE val > 20;
14781
-----+---------+----+----
14882
(0 rows)
14983

150-
/* COPY TO (partition does not exist, allowed to create partitions) */
84+
/* COPY FROM (partition does not exist, allowed to create partitions) */
15185
SET pg_pathman.enable_auto_partition = ON;
15286
COPY copy_stmt_hooking.test FROM stdin;
15387
SELECT * FROM copy_stmt_hooking.test WHERE val > 20;
@@ -194,18 +128,18 @@ WHERE attnum > 0 AND attrelid = 'copy_stmt_hooking.test_6'::REGCLASS;
194128
3
195129
(1 row)
196130

197-
/* COPY FROM (test transformed tuples) */
198-
COPY copy_stmt_hooking.test (val, c3, c4) TO stdout;
131+
/* test transformed tuples */
132+
COPY (SELECT * FROM copy_stmt_hooking.test) TO stdout;
199133
1 0 0
200134
6 0 0
201135
7 0 0
202136
11 0 0
203137
16 0 0
204138
21 0 0
205139
26 1 2
206-
/* COPY TO (insert into table with dropped column) */
140+
/* COPY FROM (insert into table with dropped column) */
207141
COPY copy_stmt_hooking.test(val, c3, c4) FROM stdin;
208-
/* COPY TO (insert into table without dropped column) */
142+
/* COPY FROM (insert into table without dropped column) */
209143
COPY copy_stmt_hooking.test(val, c3, c4) FROM stdin;
210144
/* check tuples from last partition (without dropped column) */
211145
SELECT *, tableoid::REGCLASS FROM copy_stmt_hooking.test ORDER BY val;

‎contrib/pg_pathman/sql/pathman_bgw.sql

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ DECLARE
101101
i int4 := 0; -- protect from endless loop
102102
BEGIN
103103
LOOP
104+
-- get total number of processed rows
104105
SELECT processed
105106
FROM pathman_concurrent_part_tasks
106107
WHERE relid = 'test_bgw.conc_part'::regclass
@@ -114,9 +115,12 @@ BEGIN
114115

115116
ASSERT rows IS NOT NULL;
116117

117-
-- rows should increase!
118-
IF rows_old <= rows THEN
118+
IF rows_old = rows THEN
119119
i = i + 1;
120+
ELSIF rows < rows_old THEN
121+
RAISE EXCEPTION 'rows is decreasing: new %, old %', rows, rows_old;
122+
ELSIF rows > 500 THEN
123+
RAISE EXCEPTION 'processed % rows', rows;
120124
END IF;
121125
ELSE
122126
EXIT; -- exit loop

‎contrib/pg_pathman/sql/pathman_utility_stmt.sql

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,6 @@ VACUUM FULL copy_stmt_hooking.test_2;
2828
VACUUM FULL copy_stmt_hooking.test_3;
2929
VACUUM FULL copy_stmt_hooking.test_4;
3030

31-
/* COPY TO */
32-
COPY copy_stmt_hooking.test TO stdout;
33-
\copy copy_stmt_hooking.test to stdout (format csv)
34-
\copy copy_stmt_hooking.test(comment) to stdout
35-
3631
/* DELETE ROWS, COPY FROM */
3732
DELETE FROM copy_stmt_hooking.test;
3833
COPY copy_stmt_hooking.test FROM stdin;
@@ -52,20 +47,21 @@ VACUUM FULL copy_stmt_hooking.test_2;
5247
VACUUM FULL copy_stmt_hooking.test_3;
5348
VACUUM FULL copy_stmt_hooking.test_4;
5449

55-
/* COPY FROM (specified columns) */
56-
COPY copy_stmt_hooking.test (val) TO stdout;
57-
COPY copy_stmt_hooking.test (val, comment) TO stdout;
58-
COPY copy_stmt_hooking.test (c3, val, comment) TO stdout;
59-
COPY copy_stmt_hooking.test (val, comment, c3, c4) TO stdout;
50+
/* COPY TO */
51+
COPY copy_stmt_hooking.test TO stdout; /* not ok */
52+
COPY copy_stmt_hooking.test (val) TO stdout; /* not ok */
53+
COPY (SELECT * FROM copy_stmt_hooking.test) TO stdout;
54+
COPY (SELECT * FROM copy_stmt_hooking.test) TO stdout (FORMAT CSV);
55+
\copy (SELECT * FROM copy_stmt_hooking.test) TO stdout
6056

61-
/* COPY TO (partition does not exist, NOT allowed to create partitions) */
57+
/* COPY FROM (partition does not exist, NOT allowed to create partitions) */
6258
SET pg_pathman.enable_auto_partition = OFF;
6359
COPY copy_stmt_hooking.test FROM stdin;
6460
21 test_no_part 0 0
6561
\.
6662
SELECT * FROM copy_stmt_hooking.test WHERE val > 20;
6763

68-
/* COPY TO (partition does not exist, allowed to create partitions) */
64+
/* COPY FROM (partition does not exist, allowed to create partitions) */
6965
SET pg_pathman.enable_auto_partition = ON;
7066
COPY copy_stmt_hooking.test FROM stdin;
7167
21 test_no_part 0 0
@@ -98,16 +94,16 @@ WHERE attnum > 0 AND attrelid = 'copy_stmt_hooking.test'::REGCLASS;
9894
SELECT count(*) FROM pg_attribute
9995
WHERE attnum > 0 AND attrelid = 'copy_stmt_hooking.test_6'::REGCLASS;
10096

97+
/* test transformed tuples */
98+
COPY (SELECT * FROM copy_stmt_hooking.test) TO stdout;
10199

102-
/* COPY FROM (test transformed tuples) */
103-
COPY copy_stmt_hooking.test (val, c3, c4) TO stdout;
104100

105-
/* COPY TO (insert into table with dropped column) */
101+
/* COPY FROM (insert into table with dropped column) */
106102
COPY copy_stmt_hooking.test(val, c3, c4) FROM stdin;
107103
2 1 2
108104
\.
109105

110-
/* COPY TO (insert into table without dropped column) */
106+
/* COPY FROM (insert into table without dropped column) */
111107
COPY copy_stmt_hooking.test(val, c3, c4) FROM stdin;
112108
27 1 2
113109
\.

‎contrib/pg_pathman/src/hooks.c

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,11 @@
3636
#include "utils/lsyscache.h"
3737

3838

39+
#ifdef USE_ASSERT_CHECKING
40+
#define USE_RELCACHE_LOGGING
41+
#endif
42+
43+
3944
/* Borrowed from joinpath.c */
4045
#define PATH_PARAM_BY_REL(path, rel) \
4146
((path)->param_info && bms_overlap(PATH_REQ_OUTER(path), (rel)->relids))
@@ -808,6 +813,18 @@ pathman_relcache_hook(Datum arg, Oid relid)
808813
if (!IsPathmanReady())
809814
return;
810815

816+
/* Special case: flush whole relcache */
817+
if (relid == InvalidOid)
818+
{
819+
delay_invalidation_whole_cache();
820+
821+
#ifdef USE_RELCACHE_LOGGING
822+
elog(DEBUG2, "Invalidation message for all relations [%u]", MyProcPid);
823+
#endif
824+
825+
return;
826+
}
827+
811828
/* We shouldn't even consider special OIDs */
812829
if (relid < FirstNormalObjectId)
813830
return;
@@ -827,16 +844,20 @@ pathman_relcache_hook(Datum arg, Oid relid)
827844
{
828845
delay_invalidation_parent_rel(parent_relid);
829846

847+
#ifdef USE_RELCACHE_LOGGING
830848
elog(DEBUG2, "Invalidation message for partition %u [%u]",
831849
relid, MyProcPid);
850+
#endif
832851
}
833852
/* We can't say, perform full invalidation procedure */
834853
else
835854
{
836855
delay_invalidation_vague_rel(relid);
837856

838-
elog(DEBUG2, "Invalidation message for vague relation %u [%u]",
857+
#ifdef USE_RELCACHE_LOGGING
858+
elog(DEBUG2, "Invalidation message for vague rel %u [%u]",
839859
relid, MyProcPid);
860+
#endif
840861
}
841862
}
842863

@@ -886,7 +907,7 @@ pathman_process_utility_hook(Node *first_arg,
886907
stmt_location, stmt_len, &processed);
887908
if (completionTag)
888909
snprintf(completionTag, COMPLETION_TAG_BUFSIZE,
889-
"PATHMAN COPY " UINT64_FORMAT, processed);
910+
"COPY " UINT64_FORMAT, processed);
890911

891912
return; /* don't call standard_ProcessUtility() or hooks */
892913
}

‎contrib/pg_pathman/src/include/init.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ typedef struct
5454

5555
#define PATHMAN_MCXT_COUNT 4
5656
extern MemoryContext TopPathmanContext;
57+
extern MemoryContext PathmanInvalJobsContext;
5758
extern MemoryContext PathmanRelationCacheContext;
5859
extern MemoryContext PathmanParentCacheContext;
5960
extern MemoryContext PathmanBoundCacheContext;

‎contrib/pg_pathman/src/include/relation_info.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,7 @@ const PartRelationInfo *refresh_pathman_relation_info(Oid relid,
275275
Datum *values,
276276
bool allow_incomplete);
277277
PartRelationInfo *invalidate_pathman_relation_info(Oid relid, bool *found);
278+
void invalidate_pathman_relation_info_cache(const Oid *parents, int parents_count);
278279
void remove_pathman_relation_info(Oid relid);
279280
const PartRelationInfo *get_pathman_relation_info(Oid relid);
280281
const PartRelationInfo *get_pathman_relation_info_after_lock(Oid relid,
@@ -296,6 +297,7 @@ char *canonicalize_partitioning_expression(const Oid relid,
296297

297298
/* Global invalidation routines */
298299
void delay_pathman_shutdown(void);
300+
void delay_invalidation_whole_cache(void);
299301
void delay_invalidation_parent_rel(Oid parent);
300302
void delay_invalidation_vague_rel(Oid vague_rel);
301303
void finish_delayed_invalidation(void);

0 commit comments

Comments
 (0)