Skip to content

Commit 2f9eb31

Browse files
committed
pg_dump: Allow dumping data of specific foreign servers
The new command-line switch --include-foreign-data=PATTERN lets the user specify foreign servers from which to dump foreign table data. This can be refined by further inclusion/exclusion switches, so that the user has full control over which tables to dump. A limitation is that this doesn't work in combination with parallel dumps, for implementation reasons. This might be lifted in the future, but requires shuffling some code around. Author: Luis Carril <luis.carril@swarm64.com> Reviewed-by: Daniel Gustafsson <daniel@yesql.se> Reviewed-by: Surafel Temesgen <surafel3000@gmail.com> Reviewed-by: vignesh C <vignesh21@gmail.com> Reviewed-by: Álvaro Herrera <alvherre@2ndQuadrant.com> Discussion: https://postgr.es/m/LEJPR01MB0185483C0079D2F651B16231E7FC0@LEJPR01MB0185.DEUPRD01.PROD.OUTLOOK.DE
1 parent bda6ded commit 2f9eb31

File tree

5 files changed

+185
-6
lines changed

5 files changed

+185
-6
lines changed

doc/src/sgml/ref/pg_dump.sgml

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -767,6 +767,36 @@ PostgreSQL documentation
767767
</listitem>
768768
</varlistentry>
769769

770+
<varlistentry>
771+
<term><option>--include-foreign-data=<replaceable class="parameter">foreignserver</replaceable></option></term>
772+
<listitem>
773+
<para>
774+
Dump the data for any foreign table with a foreign server
775+
matching <replaceable class="parameter">foreignserver</replaceable>
776+
pattern. Multiple foreign servers can be selected by writing multiple
777+
<option>--include-foreign-data</option> switches.
778+
Also, the <replaceable class="parameter">foreignserver</replaceable> parameter is
779+
interpreted as a pattern according to the same rules used by
780+
<application>psql</application>'s <literal>\d</literal> commands (see <xref
781+
linkend="app-psql-patterns" endterm="app-psql-patterns-title"/>),
782+
so multiple foreign servers can also be selected by writing wildcard characters
783+
in the pattern. When using wildcards, be careful to quote the pattern
784+
if needed to prevent the shell from expanding the wildcards; see
785+
<xref linkend="pg-dump-examples" endterm="pg-dump-examples-title"/>.
786+
The only exception is that an empty pattern is disallowed.
787+
</para>
788+
789+
<note>
790+
<para>
791+
When <option>--include-foreign-data</option> is specified,
792+
<application>pg_dump</application> does not check that the foreign
793+
table is writeable. Therefore, there is no guarantee that the
794+
results of a foreign table dump can be successfully restored.
795+
</para>
796+
</note>
797+
</listitem>
798+
</varlistentry>
799+
770800
<varlistentry>
771801
<term><option>--inserts</option></term>
772802
<listitem>

src/bin/pg_dump/pg_dump.c

Lines changed: 105 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,8 @@ static SimpleStringList table_exclude_patterns = {NULL, NULL};
119119
static SimpleOidList table_exclude_oids = {NULL, NULL};
120120
static SimpleStringList tabledata_exclude_patterns = {NULL, NULL};
121121
static SimpleOidList tabledata_exclude_oids = {NULL, NULL};
122+
static SimpleStringList foreign_servers_include_patterns = {NULL, NULL};
123+
static SimpleOidList foreign_servers_include_oids = {NULL, NULL};
122124

123125

124126
/* placeholders for the delimiters for comments */
@@ -153,6 +155,9 @@ static void expand_schema_name_patterns(Archive *fout,
153155
SimpleStringList *patterns,
154156
SimpleOidList *oids,
155157
bool strict_names);
158+
static void expand_foreign_server_name_patterns(Archive *fout,
159+
SimpleStringList *patterns,
160+
SimpleOidList *oids);
156161
static void expand_table_name_patterns(Archive *fout,
157162
SimpleStringList *patterns,
158163
SimpleOidList *oids,
@@ -385,6 +390,7 @@ main(int argc, char **argv)
385390
{"no-sync", no_argument, NULL, 7},
386391
{"on-conflict-do-nothing", no_argument, &dopt.do_nothing, 1},
387392
{"rows-per-insert", required_argument, NULL, 10},
393+
{"include-foreign-data", required_argument, NULL, 11},
388394

389395
{NULL, 0, NULL, 0}
390396
};
@@ -600,6 +606,11 @@ main(int argc, char **argv)
600606
dopt.dump_inserts = (int) rowsPerInsert;
601607
break;
602608

609+
case 11: /* include foreign data */
610+
simple_string_list_append(&foreign_servers_include_patterns,
611+
optarg);
612+
break;
613+
603614
default:
604615
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
605616
exit_nicely(1);
@@ -641,6 +652,12 @@ main(int argc, char **argv)
641652
exit_nicely(1);
642653
}
643654

655+
if (dopt.schemaOnly && foreign_servers_include_patterns.head != NULL)
656+
fatal("options -s/--schema-only and --include-foreign-data cannot be used together");
657+
658+
if (numWorkers > 1 && foreign_servers_include_patterns.head != NULL)
659+
fatal("option --include-foreign-data is not supported with parallel backup");
660+
644661
if (dopt.dataOnly && dopt.outputClean)
645662
{
646663
pg_log_error("options -c/--clean and -a/--data-only cannot be used together");
@@ -808,6 +825,9 @@ main(int argc, char **argv)
808825
&tabledata_exclude_oids,
809826
false);
810827

828+
expand_foreign_server_name_patterns(fout, &foreign_servers_include_patterns,
829+
&foreign_servers_include_oids);
830+
811831
/* non-matching exclusion patterns aren't an error */
812832

813833
/*
@@ -1011,6 +1031,9 @@ help(const char *progname)
10111031
printf(_(" --exclude-table-data=PATTERN do NOT dump data for the specified table(s)\n"));
10121032
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
10131033
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
1034+
printf(_(" --include-foreign-data=PATTERN\n"
1035+
" include data of foreign tables in\n"
1036+
" foreign servers matching PATTERN\n"));
10141037
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
10151038
printf(_(" --load-via-partition-root load partitions via the root table\n"));
10161039
printf(_(" --no-comments do not dump comments\n"));
@@ -1330,6 +1353,51 @@ expand_schema_name_patterns(Archive *fout,
13301353
destroyPQExpBuffer(query);
13311354
}
13321355

1356+
/*
1357+
* Find the OIDs of all foreign servers matching the given list of patterns,
1358+
* and append them to the given OID list.
1359+
*/
1360+
static void
1361+
expand_foreign_server_name_patterns(Archive *fout,
1362+
SimpleStringList *patterns,
1363+
SimpleOidList *oids)
1364+
{
1365+
PQExpBuffer query;
1366+
PGresult *res;
1367+
SimpleStringListCell *cell;
1368+
int i;
1369+
1370+
if (patterns->head == NULL)
1371+
return; /* nothing to do */
1372+
1373+
query = createPQExpBuffer();
1374+
1375+
/*
1376+
* The loop below runs multiple SELECTs might sometimes result in
1377+
* duplicate entries in the OID list, but we don't care.
1378+
*/
1379+
1380+
for (cell = patterns->head; cell; cell = cell->next)
1381+
{
1382+
appendPQExpBuffer(query,
1383+
"SELECT oid FROM pg_catalog.pg_foreign_server s\n");
1384+
processSQLNamePattern(GetConnection(fout), query, cell->val, false,
1385+
false, NULL, "s.srvname", NULL, NULL);
1386+
1387+
res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
1388+
if (PQntuples(res) == 0)
1389+
fatal("no matching foreign servers were found for pattern \"%s\"", cell->val);
1390+
1391+
for (i = 0; i < PQntuples(res); i++)
1392+
simple_oid_list_append(oids, atooid(PQgetvalue(res, i, 0)));
1393+
1394+
PQclear(res);
1395+
resetPQExpBuffer(query);
1396+
}
1397+
1398+
destroyPQExpBuffer(query);
1399+
}
1400+
13331401
/*
13341402
* Find the OIDs of all tables matching the given list of patterns,
13351403
* and append them to the given OID list. See also expand_dbname_patterns()
@@ -1775,7 +1843,6 @@ selectDumpableObject(DumpableObject *dobj, Archive *fout)
17751843
* - this routine is called by the Archiver when it wants the table
17761844
* to be dumped.
17771845
*/
1778-
17791846
static int
17801847
dumpTableData_copy(Archive *fout, void *dcontext)
17811848
{
@@ -1806,7 +1873,12 @@ dumpTableData_copy(Archive *fout, void *dcontext)
18061873
*/
18071874
column_list = fmtCopyColumnList(tbinfo, clistBuf);
18081875

1809-
if (tdinfo->filtercond)
1876+
/*
1877+
* Use COPY (SELECT ...) TO when dumping a foreign table's data, and when
1878+
* a filter condition was specified. For other cases a simple COPY
1879+
* suffices.
1880+
*/
1881+
if (tdinfo->filtercond || tbinfo->relkind == RELKIND_FOREIGN_TABLE)
18101882
{
18111883
/* Note: this syntax is only supported in 8.2 and up */
18121884
appendPQExpBufferStr(q, "COPY (SELECT ");
@@ -1818,9 +1890,10 @@ dumpTableData_copy(Archive *fout, void *dcontext)
18181890
}
18191891
else
18201892
appendPQExpBufferStr(q, "* ");
1893+
18211894
appendPQExpBuffer(q, "FROM %s %s) TO stdout;",
18221895
fmtQualifiedDumpable(tbinfo),
1823-
tdinfo->filtercond);
1896+
tdinfo->filtercond ? tdinfo->filtercond : "");
18241897
}
18251898
else
18261899
{
@@ -2336,8 +2409,11 @@ makeTableDataInfo(DumpOptions *dopt, TableInfo *tbinfo)
23362409
/* Skip VIEWs (no data to dump) */
23372410
if (tbinfo->relkind == RELKIND_VIEW)
23382411
return;
2339-
/* Skip FOREIGN TABLEs (no data to dump) */
2340-
if (tbinfo->relkind == RELKIND_FOREIGN_TABLE)
2412+
/* Skip FOREIGN TABLEs (no data to dump) unless requested explicitly */
2413+
if (tbinfo->relkind == RELKIND_FOREIGN_TABLE &&
2414+
(foreign_servers_include_oids.head == NULL ||
2415+
!simple_oid_list_member(&foreign_servers_include_oids,
2416+
tbinfo->foreign_server)))
23412417
return;
23422418
/* Skip partitioned tables (data in partitions) */
23432419
if (tbinfo->relkind == RELKIND_PARTITIONED_TABLE)
@@ -5999,6 +6075,7 @@ getTables(Archive *fout, int *numTables)
59996075
int i_toastreloptions;
60006076
int i_reloftype;
60016077
int i_relpages;
6078+
int i_foreignserver;
60026079
int i_is_identity_sequence;
60036080
int i_changed_acl;
60046081
int i_partkeydef;
@@ -6095,6 +6172,9 @@ getTables(Archive *fout, int *numTables)
60956172
"tc.relminmxid AS tminmxid, "
60966173
"c.relpersistence, c.relispopulated, "
60976174
"c.relreplident, c.relpages, am.amname, "
6175+
"CASE WHEN c.relkind = 'f' THEN "
6176+
"(SELECT ftserver FROM pg_catalog.pg_foreign_table WHERE ftrelid = c.oid) "
6177+
"ELSE 0 END AS foreignserver, "
60986178
"CASE WHEN c.reloftype <> 0 THEN c.reloftype::pg_catalog.regtype ELSE NULL END AS reloftype, "
60996179
"d.refobjid AS owning_tab, "
61006180
"d.refobjsubid AS owning_col, "
@@ -6185,6 +6265,9 @@ getTables(Archive *fout, int *numTables)
61856265
"c.relpersistence, c.relispopulated, "
61866266
"c.relreplident, c.relpages, "
61876267
"NULL AS amname, "
6268+
"CASE WHEN c.relkind = 'f' THEN "
6269+
"(SELECT ftserver FROM pg_catalog.pg_foreign_table WHERE ftrelid = c.oid) "
6270+
"ELSE 0 END AS foreignserver, "
61886271
"CASE WHEN c.reloftype <> 0 THEN c.reloftype::pg_catalog.regtype ELSE NULL END AS reloftype, "
61896272
"d.refobjid AS owning_tab, "
61906273
"d.refobjsubid AS owning_col, "
@@ -6235,6 +6318,9 @@ getTables(Archive *fout, int *numTables)
62356318
"c.relpersistence, c.relispopulated, "
62366319
"c.relreplident, c.relpages, "
62376320
"NULL AS amname, "
6321+
"CASE WHEN c.relkind = 'f' THEN "
6322+
"(SELECT ftserver FROM pg_catalog.pg_foreign_table WHERE ftrelid = c.oid) "
6323+
"ELSE 0 END AS foreignserver, "
62386324
"CASE WHEN c.reloftype <> 0 THEN c.reloftype::pg_catalog.regtype ELSE NULL END AS reloftype, "
62396325
"d.refobjid AS owning_tab, "
62406326
"d.refobjsubid AS owning_col, "
@@ -6285,6 +6371,9 @@ getTables(Archive *fout, int *numTables)
62856371
"c.relpersistence, c.relispopulated, "
62866372
"'d' AS relreplident, c.relpages, "
62876373
"NULL AS amname, "
6374+
"CASE WHEN c.relkind = 'f' THEN "
6375+
"(SELECT ftserver FROM pg_catalog.pg_foreign_table WHERE ftrelid = c.oid) "
6376+
"ELSE 0 END AS foreignserver, "
62886377
"CASE WHEN c.reloftype <> 0 THEN c.reloftype::pg_catalog.regtype ELSE NULL END AS reloftype, "
62896378
"d.refobjid AS owning_tab, "
62906379
"d.refobjsubid AS owning_col, "
@@ -6335,6 +6424,9 @@ getTables(Archive *fout, int *numTables)
63356424
"c.relpersistence, 't' as relispopulated, "
63366425
"'d' AS relreplident, c.relpages, "
63376426
"NULL AS amname, "
6427+
"CASE WHEN c.relkind = 'f' THEN "
6428+
"(SELECT ftserver FROM pg_catalog.pg_foreign_table WHERE ftrelid = c.oid) "
6429+
"ELSE 0 END AS foreignserver, "
63386430
"CASE WHEN c.reloftype <> 0 THEN c.reloftype::pg_catalog.regtype ELSE NULL END AS reloftype, "
63396431
"d.refobjid AS owning_tab, "
63406432
"d.refobjsubid AS owning_col, "
@@ -6383,6 +6475,7 @@ getTables(Archive *fout, int *numTables)
63836475
"'p' AS relpersistence, 't' as relispopulated, "
63846476
"'d' AS relreplident, c.relpages, "
63856477
"NULL AS amname, "
6478+
"NULL AS foreignserver, "
63866479
"CASE WHEN c.reloftype <> 0 THEN c.reloftype::pg_catalog.regtype ELSE NULL END AS reloftype, "
63876480
"d.refobjid AS owning_tab, "
63886481
"d.refobjsubid AS owning_col, "
@@ -6430,6 +6523,7 @@ getTables(Archive *fout, int *numTables)
64306523
"'p' AS relpersistence, 't' as relispopulated, "
64316524
"'d' AS relreplident, c.relpages, "
64326525
"NULL AS amname, "
6526+
"NULL AS foreignserver, "
64336527
"NULL AS reloftype, "
64346528
"d.refobjid AS owning_tab, "
64356529
"d.refobjsubid AS owning_col, "
@@ -6477,6 +6571,7 @@ getTables(Archive *fout, int *numTables)
64776571
"'p' AS relpersistence, 't' as relispopulated, "
64786572
"'d' AS relreplident, c.relpages, "
64796573
"NULL AS amname, "
6574+
"NULL AS foreignserver, "
64806575
"NULL AS reloftype, "
64816576
"d.refobjid AS owning_tab, "
64826577
"d.refobjsubid AS owning_col, "
@@ -6523,6 +6618,7 @@ getTables(Archive *fout, int *numTables)
65236618
"'p' AS relpersistence, 't' as relispopulated, "
65246619
"'d' AS relreplident, relpages, "
65256620
"NULL AS amname, "
6621+
"NULL AS foreignserver, "
65266622
"NULL AS reloftype, "
65276623
"d.refobjid AS owning_tab, "
65286624
"d.refobjsubid AS owning_col, "
@@ -6590,6 +6686,7 @@ getTables(Archive *fout, int *numTables)
65906686
i_relispopulated = PQfnumber(res, "relispopulated");
65916687
i_relreplident = PQfnumber(res, "relreplident");
65926688
i_relpages = PQfnumber(res, "relpages");
6689+
i_foreignserver = PQfnumber(res, "foreignserver");
65936690
i_owning_tab = PQfnumber(res, "owning_tab");
65946691
i_owning_col = PQfnumber(res, "owning_col");
65956692
i_reltablespace = PQfnumber(res, "reltablespace");
@@ -6714,6 +6811,9 @@ getTables(Archive *fout, int *numTables)
67146811
tblinfo[i].ispartition = (strcmp(PQgetvalue(res, i, i_ispartition), "t") == 0);
67156812
tblinfo[i].partbound = pg_strdup(PQgetvalue(res, i, i_partbound));
67166813

6814+
/* foreign server */
6815+
tblinfo[i].foreign_server = atooid(PQgetvalue(res, i, i_foreignserver));
6816+
67176817
/*
67186818
* Read-lock target tables to make sure they aren't DROPPED or altered
67196819
* in schema before we get around to dumping them.

src/bin/pg_dump/pg_dump.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -283,6 +283,7 @@ typedef struct _tableInfo
283283
uint32 toast_minmxid; /* toast table's relminmxid */
284284
int ncheck; /* # of CHECK expressions */
285285
char *reloftype; /* underlying type for typed table */
286+
Oid foreign_server; /* foreign server oid, if applicable */
286287
/* these two are set only if table is a sequence owned by a column: */
287288
Oid owning_tab; /* OID of table owning sequence */
288289
int owning_col; /* attr # of column owning sequence */

src/bin/pg_dump/t/001_basic.pl

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
use Config;
55
use PostgresNode;
66
use TestLib;
7-
use Test::More tests => 74;
7+
use Test::More tests => 78;
88

99
my $tempdir = TestLib::tempdir;
1010
my $tempdir_short = TestLib::tempdir_short;
@@ -49,6 +49,18 @@
4949
'pg_dump: options -s/--schema-only and -a/--data-only cannot be used together'
5050
);
5151

52+
command_fails_like(
53+
[ 'pg_dump', '-s', '--include-foreign-data=xxx' ],
54+
qr/\Qpg_dump: error: options -s\/--schema-only and --include-foreign-data cannot be used together\E/,
55+
'pg_dump: options -s/--schema-only and --include-foreign-data cannot be used together'
56+
);
57+
58+
command_fails_like(
59+
[ 'pg_dump', '-j2', '--include-foreign-data=xxx' ],
60+
qr/\Qpg_dump: error: option --include-foreign-data is not supported with parallel backup\E/,
61+
'pg_dump: option --include-foreign-data is not supported with parallel backup'
62+
);
63+
5264
command_fails_like(
5365
['pg_restore'],
5466
qr{\Qpg_restore: error: one of -d/--dbname and -f/--file must be specified\E},
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
use strict;
2+
use warnings;
3+
4+
use PostgresNode;
5+
use TestLib;
6+
use Test::More tests => 3;
7+
8+
my $tempdir = TestLib::tempdir;
9+
my $tempdir_short = TestLib::tempdir_short;
10+
11+
my $node = get_new_node('main');
12+
my $port = $node->port;
13+
14+
$node->init;
15+
$node->start;
16+
17+
#########################################
18+
# Verify that dumping foreign data includes only foreign tables of
19+
# matching servers
20+
21+
$node->safe_psql( 'postgres', "CREATE FOREIGN DATA WRAPPER dummy");
22+
$node->safe_psql( 'postgres', "CREATE SERVER s0 FOREIGN DATA WRAPPER dummy");
23+
$node->safe_psql( 'postgres', "CREATE SERVER s1 FOREIGN DATA WRAPPER dummy");
24+
$node->safe_psql( 'postgres', "CREATE SERVER s2 FOREIGN DATA WRAPPER dummy");
25+
$node->safe_psql( 'postgres', "CREATE FOREIGN TABLE t0 (a int) SERVER s0");
26+
$node->safe_psql( 'postgres', "CREATE FOREIGN TABLE t1 (a int) SERVER s1");
27+
my ($cmd, $stdout, $stderr, $result);
28+
29+
command_fails_like(
30+
[ "pg_dump", '-p', $port, 'postgres', '--include-foreign-data=s0' ],
31+
qr/foreign-data wrapper \"dummy\" has no handler\r?\npg_dump: error: query was:.*t0/,
32+
"correctly fails to dump a foreign table from a dummy FDW");
33+
34+
command_ok(
35+
[ "pg_dump", '-p', $port, 'postgres', '-a', '--include-foreign-data=s2' ] ,
36+
"dump foreign server with no tables");

0 commit comments

Comments
 (0)