Skip to content

Commit 98fe742

Browse files
michaelpqmichaelpq
committed
Extend TAP tests of pg_dump to test for compression with gzip
The test logic is extended with two new concepts: - Addition of a compression command called compress_cmd, executed between restore_cmd and dump_cmd to control the contents of the dumps. In the case of this commit, this is used to compress or decompress elements of a dump to test new code paths. - Addition of a new flag called compile_option, to check if a set of tests can be executed depending on the ./configure options used in a given build. The tests introduced here are for gzip, but they are designed so as they can easily be extended for new compression methods. Author: Georgios Kokolatos, Rachel Heaton Discussion: https://postgr.es/m/faUNEOpts9vunEaLnmxmG-DldLSg_ql137OC3JYDmgrOMHm1RvvWY2IdBkv_CRxm5spCCb_OmKNk2T03TMm0fBEWveFF9wA1WizPuAgB7Ss=@protonmail.com
1 parent 297daa9 commit 98fe742

File tree

2 files changed

+93
-2
lines changed

2 files changed

+93
-2
lines changed

‎src/bin/pg_dump/Makefile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ subdir = src/bin/pg_dump
1616
top_builddir = ../../..
1717
include $(top_builddir)/src/Makefile.global
1818

19+
export GZIP_PROGRAM=$(GZIP)
20+
1921
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
2022
LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
2123

‎src/bin/pg_dump/t/002_pg_dump.pl

Lines changed: 91 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,22 @@
2020
# test_key indicates that a given run should simply use the same
2121
# set of like/unlike tests as another run, and which run that is.
2222
#
23+
# compile_option indicates if the commands run depend on a compilation
24+
# option, if any. This can be used to control if tests should be
25+
# skipped when a build dependency is not satisfied.
26+
#
2327
# dump_cmd is the pg_dump command to run, which is an array of
2428
# the full command and arguments to run. Note that this is run
2529
# using $node->command_ok(), so the port does not need to be
2630
# specified and is pulled from $PGPORT, which is set by the
2731
# PostgreSQL::Test::Cluster system.
2832
#
33+
# compress_cmd is the utility command for (de)compression, if any.
34+
# Note that this should generally be used on pg_dump's output
35+
# either to generate a text file to run the through the tests, or
36+
# to test pg_restore's ability to parse manually compressed files
37+
# that otherwise pg_dump does not compress on its own (e.g. *.toc).
38+
#
2939
# restore_cmd is the pg_restore command to run, if any. Note
3040
# that this should generally be used when the pg_dump goes to
3141
# a non-text file and that the restore can then be used to
@@ -54,6 +64,58 @@
5464
"$tempdir/binary_upgrade.dump",
5565
],
5666
},
67+
68+
# Do not use --no-sync to give test coverage for data sync.
69+
compression_gzip_custom => {
70+
test_key => 'compression',
71+
compile_option => 'gzip',
72+
dump_cmd => [
73+
'pg_dump', '--format=custom',
74+
'--compress=1', "--file=$tempdir/compression_gzip_custom.dump",
75+
'postgres',
76+
],
77+
restore_cmd => [
78+
'pg_restore',
79+
"--file=$tempdir/compression_gzip_custom.sql",
80+
"$tempdir/compression_gzip_custom.dump",
81+
],
82+
},
83+
84+
# Do not use --no-sync to give test coverage for data sync.
85+
compression_gzip_dir => {
86+
test_key => 'compression',
87+
compile_option => 'gzip',
88+
dump_cmd => [
89+
'pg_dump', '--jobs=2',
90+
'--format=directory', '--compress=1',
91+
"--file=$tempdir/compression_gzip_dir", 'postgres',
92+
],
93+
# Give coverage for manually compressed blob.toc files during
94+
# restore.
95+
compress_cmd => {
96+
program => $ENV{'GZIP_PROGRAM'},
97+
args => [ '-f', "$tempdir/compression_gzip_dir/blobs.toc", ],
98+
},
99+
restore_cmd => [
100+
'pg_restore', '--jobs=2',
101+
"--file=$tempdir/compression_gzip_dir.sql",
102+
"$tempdir/compression_gzip_dir",
103+
],
104+
},
105+
106+
compression_gzip_plain => {
107+
test_key => 'compression',
108+
compile_option => 'gzip',
109+
dump_cmd => [
110+
'pg_dump', '--format=plain', '-Z1',
111+
"--file=$tempdir/compression_gzip_plain.sql.gz", 'postgres',
112+
],
113+
# Decompress the generated file to run through the tests.
114+
compress_cmd => {
115+
program => $ENV{'GZIP_PROGRAM'},
116+
args => [ '-d', "$tempdir/compression_gzip_plain.sql.gz", ],
117+
},
118+
},
57119
clean => {
58120
dump_cmd => [
59121
'pg_dump',
@@ -424,6 +486,7 @@
424486
binary_upgrade => 1,
425487
clean => 1,
426488
clean_if_exists => 1,
489+
compression => 1,
427490
createdb => 1,
428491
defaults => 1,
429492
exclude_dump_test_schema => 1,
@@ -3098,6 +3161,7 @@
30983161
binary_upgrade => 1,
30993162
clean => 1,
31003163
clean_if_exists => 1,
3164+
compression => 1,
31013165
createdb => 1,
31023166
defaults => 1,
31033167
exclude_test_table => 1,
@@ -3171,6 +3235,7 @@
31713235
binary_upgrade => 1,
31723236
clean => 1,
31733237
clean_if_exists => 1,
3238+
compression => 1,
31743239
createdb => 1,
31753240
defaults => 1,
31763241
exclude_dump_test_schema => 1,
@@ -3833,8 +3898,9 @@
38333898
$collation_support = 1;
38343899
}
38353900
3836-
# Determine whether build supports LZ4.
3837-
my $supports_lz4 = check_pg_config("#define USE_LZ4 1");
3901+
# Determine whether build supports LZ4 and gzip.
3902+
my $supports_lz4 = check_pg_config("#define USE_LZ4 1");
3903+
my $supports_gzip = check_pg_config("#define HAVE_LIBZ 1");
38383904
38393905
# Create additional databases for mutations of schema public
38403906
$node->psql('postgres', 'create database regress_pg_dump_test;');
@@ -3947,9 +4013,32 @@
39474013
my $test_key = $run;
39484014
my $run_db = 'postgres';
39494015
4016+
# Skip command-level tests for gzip if there is no support for it.
4017+
if ( defined($pgdump_runs{$run}->{compile_option})
4018+
&& $pgdump_runs{$run}->{compile_option} eq 'gzip'
4019+
&& !$supports_gzip)
4020+
{
4021+
note "$run: skipped due to no gzip support";
4022+
next;
4023+
}
4024+
39504025
$node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} },
39514026
"$run: pg_dump runs");
39524027
4028+
if ($pgdump_runs{$run}->{compress_cmd})
4029+
{
4030+
my ($compress_cmd) = $pgdump_runs{$run}->{compress_cmd};
4031+
my $compress_program = $compress_cmd->{program};
4032+
4033+
# Skip the rest of the test if the compression program is
4034+
# not defined.
4035+
next if (!defined($compress_program) || $compress_program eq '');
4036+
4037+
my @full_compress_cmd =
4038+
($compress_cmd->{program}, @{ $compress_cmd->{args} });
4039+
command_ok(\@full_compress_cmd, "$run: compression commands");
4040+
}
4041+
39534042
if ($pgdump_runs{$run}->{restore_cmd})
39544043
{
39554044
$node->command_ok(\@{ $pgdump_runs{$run}->{restore_cmd} },

0 commit comments

Comments
 (0)