Skip to content

Commit d62bcc8

Browse files
michaelpqmichaelpq
committed
Rework compression options of pg_receivewal
pg_receivewal includes since cada1af the option --compress, to allow the compression of WAL segments using gzip, with a value of 0 (the default) meaning that no compression can be used. This commit introduces a new option, called --compression-method, able to use as values "none", the default, and "gzip", to make things more extensible. The case of --compress=0 becomes fuzzy with this option layer, so we have made the choice to make pg_receivewal return an error when using "none" and a non-zero compression level, meaning that the authorized values of --compress are now [1,9] instead of [0,9]. Not specifying --compress with "gzip" as compression method makes pg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION). The code in charge of finding the streaming start LSN when scanning the existing archives is refactored and made more extensible. While on it, rename "compression" to "compression_level" in walmethods.c, to reduce the confusion with the introduction of the compression method, even if the tar method used by pg_basebackup does not rely on the compression method (yet, at least), but just on the compression level (this area could be improved more, actually). This is in preparation for an upcoming patch that adds LZ4 support to pg_receivewal. Author: Georgios Kokolatos Reviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar, Robert Haas Discussion: https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
1 parent 581055c commit d62bcc8

File tree

8 files changed

+231
-103
lines changed

8 files changed

+231
-103
lines changed

‎doc/src/sgml/ref/pg_receivewal.sgml

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -263,15 +263,36 @@ PostgreSQL documentation
263263
</listitem>
264264
</varlistentry>
265265

266+
<varlistentry>
267+
<term><option>--compression-method=<replaceable class="parameter">level</replaceable></option></term>
268+
<listitem>
269+
<para>
270+
Enables compression of write-ahead logs using the specified method.
271+
Supported values <literal>gzip</literal>, and
272+
<literal>none</literal>.
273+
</para>
274+
275+
<para>
276+
The suffix <filename>.gz</filename> will automatically be added to
277+
all filenames when using <literal>gzip</literal>
278+
</para>
279+
</listitem>
280+
</varlistentry>
281+
266282
<varlistentry>
267283
<term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
268284
<term><option>--compress=<replaceable class="parameter">level</replaceable></option></term>
269285
<listitem>
270286
<para>
271-
Enables gzip compression of write-ahead logs, and specifies the
272-
compression level (0 through 9, 0 being no compression and 9 being best
273-
compression). The suffix <filename>.gz</filename> will
274-
automatically be added to all filenames.
287+
Specifies the compression level (<literal>1</literal> through
288+
<literal>9</literal>, <literal>1</literal> being worst compression
289+
and <literal>9</literal> being best compression) for WAL segments
290+
compressed with <application>gzip</application>.
291+
</para>
292+
293+
<para>
294+
This option requires <option>--compression-method</option> to be
295+
specified with <literal>gzip</literal>.
275296
</para>
276297
</listitem>
277298
</varlistentry>

‎src/bin/pg_basebackup/pg_basebackup.c

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -555,10 +555,13 @@ LogStreamerMain(logstreamer_param *param)
555555
stream.replication_slot = replication_slot;
556556

557557
if (format == 'p')
558-
stream.walmethod = CreateWalDirectoryMethod(param->xlog, 0,
558+
stream.walmethod = CreateWalDirectoryMethod(param->xlog,
559+
COMPRESSION_NONE, 0,
559560
stream.do_sync);
560561
else
561-
stream.walmethod = CreateWalTarMethod(param->xlog, compresslevel,
562+
stream.walmethod = CreateWalTarMethod(param->xlog,
563+
COMPRESSION_NONE, /* ignored */
564+
compresslevel,
562565
stream.do_sync);
563566

564567
if (!ReceiveXlogStream(param->bgconn, &stream))

‎src/bin/pg_basebackup/pg_receivewal.c

Lines changed: 117 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@
1919
#include <signal.h>
2020
#include <sys/stat.h>
2121
#include <unistd.h>
22+
#ifdef HAVE_LIBZ
23+
#include <zlib.h>
24+
#endif
2225

2326
#include "access/xlog_internal.h"
2427
#include "common/file_perm.h"
@@ -45,6 +48,7 @@ static bool do_drop_slot = false;
4548
static bool do_sync = true;
4649
static bool synchronous = false;
4750
static char *replication_slot = NULL;
51+
static WalCompressionMethod compression_method = COMPRESSION_NONE;
4852
static XLogRecPtr endpos = InvalidXLogRecPtr;
4953

5054

@@ -63,16 +67,6 @@ disconnect_atexit(void)
6367
PQfinish(conn);
6468
}
6569

66-
/* Routines to evaluate segment file format */
67-
#define IsCompressXLogFileName(fname) \
68-
(strlen(fname) == XLOG_FNAME_LEN + strlen(".gz") && \
69-
strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \
70-
strcmp((fname) + XLOG_FNAME_LEN, ".gz") == 0)
71-
#define IsPartialCompressXLogFileName(fname) \
72-
(strlen(fname) == XLOG_FNAME_LEN + strlen(".gz.partial") && \
73-
strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \
74-
strcmp((fname) + XLOG_FNAME_LEN, ".gz.partial") == 0)
75-
7670
static void
7771
usage(void)
7872
{
@@ -92,7 +86,9 @@ usage(void)
9286
printf(_(" --synchronous flush write-ahead log immediately after writing\n"));
9387
printf(_(" -v, --verbose output verbose messages\n"));
9488
printf(_(" -V, --version output version information, then exit\n"));
95-
printf(_(" -Z, --compress=0-9 compress logs with given compression level\n"));
89+
printf(_(" --compression-method=METHOD\n"
90+
" method to compress logs\n"));
91+
printf(_(" -Z, --compress=1-9 compress logs with given compression level\n"));
9692
printf(_(" -?, --help show this help, then exit\n"));
9793
printf(_("\nConnection options:\n"));
9894
printf(_(" -d, --dbname=CONNSTR connection string\n"));
@@ -108,6 +104,60 @@ usage(void)
108104
printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
109105
}
110106

107+
/*
108+
* Check if the filename looks like a WAL file, letting caller know if this
109+
* WAL segment is partial and/or compressed.
110+
*/
111+
static bool
112+
is_xlogfilename(const char *filename, bool *ispartial,
113+
WalCompressionMethod *wal_compression_method)
114+
{
115+
size_t fname_len = strlen(filename);
116+
size_t xlog_pattern_len = strspn(filename, "0123456789ABCDEF");
117+
118+
/* File does not look like a WAL file */
119+
if (xlog_pattern_len != XLOG_FNAME_LEN)
120+
return false;
121+
122+
/* File looks like a completed uncompressed WAL file */
123+
if (fname_len == XLOG_FNAME_LEN)
124+
{
125+
*ispartial = false;
126+
*wal_compression_method = COMPRESSION_NONE;
127+
return true;
128+
}
129+
130+
/* File looks like a completed gzip-compressed WAL file */
131+
if (fname_len == XLOG_FNAME_LEN + strlen(".gz") &&
132+
strcmp(filename + XLOG_FNAME_LEN, ".gz") == 0)
133+
{
134+
*ispartial = false;
135+
*wal_compression_method = COMPRESSION_GZIP;
136+
return true;
137+
}
138+
139+
/* File looks like a partial uncompressed WAL file */
140+
if (fname_len == XLOG_FNAME_LEN + strlen(".partial") &&
141+
strcmp(filename + XLOG_FNAME_LEN, ".partial") == 0)
142+
{
143+
*ispartial = true;
144+
*wal_compression_method = COMPRESSION_NONE;
145+
return true;
146+
}
147+
148+
/* File looks like a partial gzip-compressed WAL file */
149+
if (fname_len == XLOG_FNAME_LEN + strlen(".gz.partial") &&
150+
strcmp(filename + XLOG_FNAME_LEN, ".gz.partial") == 0)
151+
{
152+
*ispartial = true;
153+
*wal_compression_method = COMPRESSION_GZIP;
154+
return true;
155+
}
156+
157+
/* File does not look like something we know */
158+
return false;
159+
}
160+
111161
static bool
112162
stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished)
113163
{
@@ -213,33 +263,11 @@ FindStreamingStart(uint32 *tli)
213263
{
214264
uint32 tli;
215265
XLogSegNo segno;
266+
WalCompressionMethod wal_compression_method;
216267
bool ispartial;
217-
bool iscompress;
218268

219-
/*
220-
* Check if the filename looks like an xlog file, or a .partial file.
221-
*/
222-
if (IsXLogFileName(dirent->d_name))
223-
{
224-
ispartial = false;
225-
iscompress = false;
226-
}
227-
else if (IsPartialXLogFileName(dirent->d_name))
228-
{
229-
ispartial = true;
230-
iscompress = false;
231-
}
232-
else if (IsCompressXLogFileName(dirent->d_name))
233-
{
234-
ispartial = false;
235-
iscompress = true;
236-
}
237-
else if (IsPartialCompressXLogFileName(dirent->d_name))
238-
{
239-
ispartial = true;
240-
iscompress = true;
241-
}
242-
else
269+
if (!is_xlogfilename(dirent->d_name,
270+
&ispartial, &wal_compression_method))
243271
continue;
244272

245273
/*
@@ -250,14 +278,14 @@ FindStreamingStart(uint32 *tli)
250278
/*
251279
* Check that the segment has the right size, if it's supposed to be
252280
* completed. For non-compressed segments just check the on-disk size
253-
* and see if it matches a completed segment. For compressed segments,
254-
* look at the last 4 bytes of the compressed file, which is where the
255-
* uncompressed size is located for gz files with a size lower than
256-
* 4GB, and then compare it to the size of a completed segment. The 4
257-
* last bytes correspond to the ISIZE member according to
281+
* and see if it matches a completed segment. For gzip-compressed
282+
* segments, look at the last 4 bytes of the compressed file, which is
283+
* where the uncompressed size is located for files with a size lower
284+
* than 4GB, and then compare it to the size of a completed segment.
285+
* The 4 last bytes correspond to the ISIZE member according to
258286
* http://www.zlib.org/rfc-gzip.html.
259287
*/
260-
if (!ispartial && !iscompress)
288+
if (!ispartial && wal_compression_method == COMPRESSION_NONE)
261289
{
262290
struct stat statbuf;
263291
char fullpath[MAXPGPATH * 2];
@@ -276,7 +304,7 @@ FindStreamingStart(uint32 *tli)
276304
continue;
277305
}
278306
}
279-
else if (!ispartial && iscompress)
307+
else if (!ispartial && wal_compression_method == COMPRESSION_GZIP)
280308
{
281309
int fd;
282310
char buf[4];
@@ -457,7 +485,9 @@ StreamLog(void)
457485
stream.synchronous = synchronous;
458486
stream.do_sync = do_sync;
459487
stream.mark_done = false;
460-
stream.walmethod = CreateWalDirectoryMethod(basedir, compresslevel,
488+
stream.walmethod = CreateWalDirectoryMethod(basedir,
489+
compression_method,
490+
compresslevel,
461491
stream.do_sync);
462492
stream.partial_suffix = ".partial";
463493
stream.replication_slot = replication_slot;
@@ -510,6 +540,7 @@ main(int argc, char **argv)
510540
{"status-interval", required_argument, NULL, 's'},
511541
{"slot", required_argument, NULL, 'S'},
512542
{"verbose", no_argument, NULL, 'v'},
543+
{"compression-method", required_argument, NULL, 'I'},
513544
{"compress", required_argument, NULL, 'Z'},
514545
/* action */
515546
{"create-slot", no_argument, NULL, 1},
@@ -595,8 +626,20 @@ main(int argc, char **argv)
595626
case 'v':
596627
verbose++;
597628
break;
629+
case 'I':
630+
if (pg_strcasecmp(optarg, "gzip") == 0)
631+
compression_method = COMPRESSION_GZIP;
632+
else if (pg_strcasecmp(optarg, "none") == 0)
633+
compression_method = COMPRESSION_NONE;
634+
else
635+
{
636+
pg_log_error("invalid value \"%s\" for option %s",
637+
optarg, "--compress-method");
638+
exit(1);
639+
}
640+
break;
598641
case 'Z':
599-
if (!option_parse_int(optarg, "-Z/--compress", 0, 9,
642+
if (!option_parse_int(optarg, "-Z/--compress", 1, 9,
600643
&compresslevel))
601644
exit(1);
602645
break;
@@ -676,13 +719,37 @@ main(int argc, char **argv)
676719
exit(1);
677720
}
678721

679-
#ifndef HAVE_LIBZ
680-
if (compresslevel != 0)
722+
723+
/*
724+
* Compression-related options.
725+
*/
726+
switch (compression_method)
681727
{
682-
pg_log_error("this build does not support compression");
683-
exit(1);
684-
}
728+
case COMPRESSION_NONE:
729+
if (compresslevel != 0)
730+
{
731+
pg_log_error("cannot use --compress with --compression-method=%s",
732+
"none");
733+
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
734+
progname);
735+
exit(1);
736+
}
737+
break;
738+
case COMPRESSION_GZIP:
739+
#ifdef HAVE_LIBZ
740+
if (compresslevel == 0)
741+
{
742+
pg_log_info("no value specified for --compress, switching to default");
743+
compresslevel = Z_DEFAULT_COMPRESSION;
744+
}
745+
#else
746+
pg_log_error("this build does not support compression with %s",
747+
"gzip");
748+
exit(1);
685749
#endif
750+
break;
751+
}
752+
686753

687754
/*
688755
* Check existence of destination folder.

‎src/bin/pg_basebackup/receivelog.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint)
109109
* When streaming to tar, no file with this name will exist before, so we
110110
* never have to verify a size.
111111
*/
112-
if (stream->walmethod->compression() == 0 &&
112+
if (stream->walmethod->compression_method() == COMPRESSION_NONE &&
113113
stream->walmethod->existsfile(fn))
114114
{
115115
size = stream->walmethod->get_file_size(fn);

‎src/bin/pg_basebackup/t/020_pg_receivewal.pl

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
use warnings;
66
use PostgreSQL::Test::Utils;
77
use PostgreSQL::Test::Cluster;
8-
use Test::More tests => 35;
8+
use Test::More tests => 37;
99

1010
program_help_ok('pg_receivewal');
1111
program_version_ok('pg_receivewal');
@@ -33,6 +33,13 @@
3333
$primary->command_fails(
3434
[ 'pg_receivewal', '-D', $stream_dir, '--synchronous', '--no-sync' ],
3535
'failure if --synchronous specified with --no-sync');
36+
$primary->command_fails_like(
37+
[
38+
'pg_receivewal', '-D', $stream_dir, '--compression-method', 'none',
39+
'--compress', '1'
40+
],
41+
qr/\Qpg_receivewal: error: cannot use --compress with --compression-method=none/,
42+
'failure if --compress spwcified with --compression-method=none');
3643

3744
# Slot creation and drop
3845
my $slot_name = 'test';
@@ -90,8 +97,11 @@
9097
# a valid value.
9198
$primary->command_ok(
9299
[
93-
'pg_receivewal', '-D', $stream_dir, '--verbose',
94-
'--endpos', $nextlsn, '--compress', '1 ',
100+
'pg_receivewal', '-D',
101+
$stream_dir, '--verbose',
102+
'--endpos', $nextlsn,
103+
'--compression-method', 'gzip',
104+
'--compress', '1 ',
95105
'--no-loop'
96106
],
97107
"streaming some WAL using ZLIB compression");

0 commit comments

Comments
 (0)