Skip to content

Commit d07c294

Browse files
michaelpqmichaelpq
committed
Add support for progress reporting to pg_verifybackup
This adds a new option to pg_verifybackup called -P/--progress, showing every second some information about the progress of the checksum verification based on the data of a backup manifest. Similarly to what is done for pg_rewind and pg_basebackup, the information printed in the progress report consists of the current amount of data computed and the total amount of data that will be computed. Note that files found with an incorrect size do not have their checksum verified, hence their size is not appended to the total amount of data estimated during the first scan of the manifest data (such incorrect sizes could be overly high, for one, falsifying the progress report). Author: Masahiko Sawada Discussion: https://postgr.es/m/CAD21AoC5+JOgMd4o3z_oxw0f8JDSsCYY7zSbhe-O9x7f33rw_A@mail.gmail.com
1 parent 71c3779 commit d07c294

File tree

3 files changed

+111
-7
lines changed

3 files changed

+111
-7
lines changed

‎doc/src/sgml/ref/pg_verifybackup.sgml

+15
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,21 @@ PostgreSQL documentation
178178
</listitem>
179179
</varlistentry>
180180

181+
<varlistentry>
182+
<term><option>-P</option></term>
183+
<term><option>--progress</option></term>
184+
<listitem>
185+
<para>
186+
Enable progress reporting. Turning this on will deliver a progress
187+
report while verifying checksums.
188+
</para>
189+
<para>
190+
This option cannot be used together with the option
191+
<option>--quiet</option>.
192+
</para>
193+
</listitem>
194+
</varlistentry>
195+
181196
<varlistentry>
182197
<term><option>-q</option></term>
183198
<term><option>--quiet</option></term>

‎src/bin/pg_verifybackup/pg_verifybackup.c

+83-3
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,14 @@
1616
#include <dirent.h>
1717
#include <fcntl.h>
1818
#include <sys/stat.h>
19+
#include <time.h>
1920

2021
#include "common/hashfn.h"
2122
#include "common/logging.h"
2223
#include "fe_utils/simple_list.h"
2324
#include "getopt_long.h"
2425
#include "parse_manifest.h"
26+
#include "pgtime.h"
2527

2628
/*
2729
* For efficiency, we'd like our hash table containing information about the
@@ -58,6 +60,9 @@ typedef struct manifest_file
5860
bool bad;
5961
} manifest_file;
6062

63+
#define should_verify_checksum(m) \
64+
(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
65+
6166
/*
6267
* Define a hash table which we can use to store information about the files
6368
* mentioned in the backup manifest.
@@ -147,10 +152,19 @@ static void report_fatal_error(const char *pg_restrict fmt,...)
147152
pg_attribute_printf(1, 2) pg_attribute_noreturn();
148153
static bool should_ignore_relpath(verifier_context *context, char *relpath);
149154

155+
static void progress_report(bool finished);
150156
static void usage(void);
151157

152158
static const char *progname;
153159

160+
/* options */
161+
static bool show_progress = false;
162+
static bool skip_checksums = false;
163+
164+
/* Progress indicators */
165+
static uint64 total_size = 0;
166+
static uint64 done_size = 0;
167+
154168
/*
155169
* Main entry point.
156170
*/
@@ -162,6 +176,7 @@ main(int argc, char **argv)
162176
{"ignore", required_argument, NULL, 'i'},
163177
{"manifest-path", required_argument, NULL, 'm'},
164178
{"no-parse-wal", no_argument, NULL, 'n'},
179+
{"progress", no_argument, NULL, 'P'},
165180
{"quiet", no_argument, NULL, 'q'},
166181
{"skip-checksums", no_argument, NULL, 's'},
167182
{"wal-directory", required_argument, NULL, 'w'},
@@ -174,7 +189,6 @@ main(int argc, char **argv)
174189
char *manifest_path = NULL;
175190
bool no_parse_wal = false;
176191
bool quiet = false;
177-
bool skip_checksums = false;
178192
char *wal_directory = NULL;
179193
char *pg_waldump_path = NULL;
180194

@@ -219,7 +233,7 @@ main(int argc, char **argv)
219233
simple_string_list_append(&context.ignore_list, "recovery.signal");
220234
simple_string_list_append(&context.ignore_list, "standby.signal");
221235

222-
while ((c = getopt_long(argc, argv, "ei:m:nqsw:", long_options, NULL)) != -1)
236+
while ((c = getopt_long(argc, argv, "ei:m:nPqsw:", long_options, NULL)) != -1)
223237
{
224238
switch (c)
225239
{
@@ -241,6 +255,9 @@ main(int argc, char **argv)
241255
case 'n':
242256
no_parse_wal = true;
243257
break;
258+
case 'P':
259+
show_progress = true;
260+
break;
244261
case 'q':
245262
quiet = true;
246263
break;
@@ -277,6 +294,11 @@ main(int argc, char **argv)
277294
exit(1);
278295
}
279296

297+
/* Complain if the specified arguments conflict */
298+
if (show_progress && quiet)
299+
pg_fatal("cannot specify both %s and %s",
300+
"-P/--progress", "-q/--quiet");
301+
280302
/* Unless --no-parse-wal was specified, we will need pg_waldump. */
281303
if (!no_parse_wal)
282304
{
@@ -638,6 +660,10 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
638660
m->bad = true;
639661
}
640662

663+
/* Update statistics for progress report, if necessary */
664+
if (show_progress && !skip_checksums && should_verify_checksum(m))
665+
total_size += m->size;
666+
641667
/*
642668
* We don't verify checksums at this stage. We first finish verifying that
643669
* we have the expected set of files with the expected sizes, and only
@@ -675,10 +701,12 @@ verify_backup_checksums(verifier_context *context)
675701
manifest_files_iterator it;
676702
manifest_file *m;
677703

704+
progress_report(false);
705+
678706
manifest_files_start_iterate(context->ht, &it);
679707
while ((m = manifest_files_iterate(context->ht, &it)) != NULL)
680708
{
681-
if (m->matched && !m->bad && m->checksum_type != CHECKSUM_TYPE_NONE &&
709+
if (should_verify_checksum(m) &&
682710
!should_ignore_relpath(context, m->pathname))
683711
{
684712
char *fullpath;
@@ -694,6 +722,8 @@ verify_backup_checksums(verifier_context *context)
694722
pfree(fullpath);
695723
}
696724
}
725+
726+
progress_report(true);
697727
}
698728

699729
/*
@@ -740,6 +770,10 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
740770
close(fd);
741771
return;
742772
}
773+
774+
/* Report progress */
775+
done_size += rc;
776+
progress_report(false);
743777
}
744778
if (rc < 0)
745779
report_backup_error(context, "could not read file \"%s\": %m",
@@ -894,6 +928,51 @@ hash_string_pointer(char *s)
894928
return hash_bytes(ss, strlen(s));
895929
}
896930

931+
/*
932+
* Print a progress report based on the global variables.
933+
*
934+
* Progress report is written at maximum once per second, unless the finished
935+
* parameter is set to true.
936+
*
937+
* If finished is set to true, this is the last progress report. The cursor
938+
* is moved to the next line.
939+
*/
940+
static void
941+
progress_report(bool finished)
942+
{
943+
static pg_time_t last_progress_report = 0;
944+
pg_time_t now;
945+
int percent_size = 0;
946+
char totalsize_str[32];
947+
char donesize_str[32];
948+
949+
if (!show_progress)
950+
return;
951+
952+
now = time(NULL);
953+
if (now == last_progress_report && !finished)
954+
return; /* Max once per second */
955+
956+
last_progress_report = now;
957+
percent_size = total_size ? (int) ((done_size * 100 / total_size)) : 0;
958+
959+
snprintf(totalsize_str, sizeof(totalsize_str), UINT64_FORMAT,
960+
total_size / 1024);
961+
snprintf(donesize_str, sizeof(donesize_str), UINT64_FORMAT,
962+
done_size / 1024);
963+
964+
fprintf(stderr,
965+
_("%*s/%s kB (%d%%) verified"),
966+
(int) strlen(totalsize_str),
967+
donesize_str, totalsize_str, percent_size);
968+
969+
/*
970+
* Stay on the same line if reporting to a terminal and we're not done
971+
* yet.
972+
*/
973+
fputc((!finished && isatty(fileno(stderr))) ? '\r' : '\n', stderr);
974+
}
975+
897976
/*
898977
* Print out usage information and exit.
899978
*/
@@ -907,6 +986,7 @@ usage(void)
907986
printf(_(" -i, --ignore=RELATIVE_PATH ignore indicated path\n"));
908987
printf(_(" -m, --manifest-path=PATH use specified path for manifest\n"));
909988
printf(_(" -n, --no-parse-wal do not try to parse WAL files\n"));
989+
printf(_(" -P, --progress show progress information\n"));
910990
printf(_(" -q, --quiet do not print any output, except for errors\n"));
911991
printf(_(" -s, --skip-checksums skip checksum verification\n"));
912992
printf(_(" -w, --wal-directory=PATH use specified path for WAL files\n"));

‎src/bin/pg_verifybackup/t/004_options.pl

+13-4
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,12 @@
2828
is($stdout, '', "-q succeeds: no stdout");
2929
is($stderr, '', "-q succeeds: no stderr");
3030

31+
# Test invalid options
32+
command_fails_like(
33+
[ 'pg_verifybackup', '--progress', '--quiet', $backup_path ],
34+
qr{cannot specify both -P/--progress and -q/--quiet},
35+
'cannot use --progress and --quiet at the same time');
36+
3137
# Corrupt the PG_VERSION file.
3238
my $version_pathname = "$backup_path/PG_VERSION";
3339
my $version_contents = slurp_file($version_pathname);
@@ -48,10 +54,13 @@
4854
qr/backup successfully verified/,
4955
'-s skips checksumming');
5056

51-
# Validation should succeed if we ignore the problem file.
52-
command_like(
53-
[ 'pg_verifybackup', '-i', 'PG_VERSION', $backup_path ],
54-
qr/backup successfully verified/,
57+
# Validation should succeed if we ignore the problem file. Also, check
58+
# the progress information.
59+
command_checks_all(
60+
[ 'pg_verifybackup', '--progress', '-i', 'PG_VERSION', $backup_path ],
61+
0,
62+
[qr/backup successfully verified/],
63+
[qr{(\d+/\d+ kB \(\d+%\) verified)+}],
5564
'-i ignores problem file');
5665

5766
# PG_VERSION is already corrupt; let's try also removing all of pg_xact.

0 commit comments

Comments
 (0)