Skip to content

Commit d450994

Browse files
robertmhaasrobertmhaas
committed
Allow server-side compression to be used with -Fp.
If you have a low-bandwidth connection between the client and the server, it's reasonable to want to compress on the server side but then decompress and extract the backup on the client side. This commit allows you do to do just that. Dipesh Pandit, with minor and mostly cosmetic changes by me. Discussion: http://postgr.es/m/CAN1g5_HiSh8ajUMd4ePtGyCXo89iKZTzaNyzP_qv1eJbi4YHXA@mail.gmail.com
1 parent 43f33dc commit d450994

File tree

7 files changed

+463
-188
lines changed

7 files changed

+463
-188
lines changed

‎doc/src/sgml/ref/pg_basebackup.sgml

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -428,8 +428,11 @@ PostgreSQL documentation
428428
</para>
429429
<para>
430430
When the tar format is used, the suffix <filename>.gz</filename> will
431-
automatically be added to all tar filenames. Compression is not
432-
available in plain format.
431+
automatically be added to all tar filenames. When the plain format is
432+
used, client-side compression may not be specified, but it is
433+
still possible to request server-side compression. If this is done,
434+
the server will compress the backup for transmission, and the
435+
client will decompress and extract it.
433436
</para>
434437
</listitem>
435438
</varlistentry>

‎src/bin/pg_basebackup/Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ OBJS = \
3838
BBOBJS = \
3939
pg_basebackup.o \
4040
bbstreamer_file.o \
41+
bbstreamer_gzip.o \
4142
bbstreamer_inject.o \
4243
bbstreamer_tar.o
4344

‎src/bin/pg_basebackup/bbstreamer.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,7 @@ extern bbstreamer *bbstreamer_extractor_new(const char *basepath,
205205
const char *(*link_map) (const char *),
206206
void (*report_output_file) (const char *));
207207

208+
extern bbstreamer *bbstreamer_gzip_decompressor_new(bbstreamer *next);
208209
extern bbstreamer *bbstreamer_tar_parser_new(bbstreamer *next);
209210
extern bbstreamer *bbstreamer_tar_terminator_new(bbstreamer *next);
210211
extern bbstreamer *bbstreamer_tar_archiver_new(bbstreamer *next);

‎src/bin/pg_basebackup/bbstreamer_file.c

Lines changed: 0 additions & 182 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,6 @@
1111

1212
#include "postgres_fe.h"
1313

14-
#ifdef HAVE_LIBZ
15-
#include <zlib.h>
16-
#endif
17-
1814
#include <unistd.h>
1915

2016
#include "bbstreamer.h"
@@ -30,15 +26,6 @@ typedef struct bbstreamer_plain_writer
3026
bool should_close_file;
3127
} bbstreamer_plain_writer;
3228

33-
#ifdef HAVE_LIBZ
34-
typedef struct bbstreamer_gzip_writer
35-
{
36-
bbstreamer base;
37-
char *pathname;
38-
gzFile gzfile;
39-
} bbstreamer_gzip_writer;
40-
#endif
41-
4229
typedef struct bbstreamer_extractor
4330
{
4431
bbstreamer base;
@@ -62,22 +49,6 @@ const bbstreamer_ops bbstreamer_plain_writer_ops = {
6249
.free = bbstreamer_plain_writer_free
6350
};
6451

65-
#ifdef HAVE_LIBZ
66-
static void bbstreamer_gzip_writer_content(bbstreamer *streamer,
67-
bbstreamer_member *member,
68-
const char *data, int len,
69-
bbstreamer_archive_context context);
70-
static void bbstreamer_gzip_writer_finalize(bbstreamer *streamer);
71-
static void bbstreamer_gzip_writer_free(bbstreamer *streamer);
72-
static const char *get_gz_error(gzFile gzf);
73-
74-
const bbstreamer_ops bbstreamer_gzip_writer_ops = {
75-
.content = bbstreamer_gzip_writer_content,
76-
.finalize = bbstreamer_gzip_writer_finalize,
77-
.free = bbstreamer_gzip_writer_free
78-
};
79-
#endif
80-
8152
static void bbstreamer_extractor_content(bbstreamer *streamer,
8253
bbstreamer_member *member,
8354
const char *data, int len,
@@ -195,159 +166,6 @@ bbstreamer_plain_writer_free(bbstreamer *streamer)
195166
pfree(mystreamer);
196167
}
197168

198-
/*
199-
* Create a bbstreamer that just compresses data using gzip, and then writes
200-
* it to a file.
201-
*
202-
* As in the case of bbstreamer_plain_writer_new, pathname is always used
203-
* for error reporting purposes; if file is NULL, it is also the opened and
204-
* closed so that the data may be written there.
205-
*/
206-
bbstreamer *
207-
bbstreamer_gzip_writer_new(char *pathname, FILE *file, int compresslevel)
208-
{
209-
#ifdef HAVE_LIBZ
210-
bbstreamer_gzip_writer *streamer;
211-
212-
streamer = palloc0(sizeof(bbstreamer_gzip_writer));
213-
*((const bbstreamer_ops **) &streamer->base.bbs_ops) =
214-
&bbstreamer_gzip_writer_ops;
215-
216-
streamer->pathname = pstrdup(pathname);
217-
218-
if (file == NULL)
219-
{
220-
streamer->gzfile = gzopen(pathname, "wb");
221-
if (streamer->gzfile == NULL)
222-
{
223-
pg_log_error("could not create compressed file \"%s\": %m",
224-
pathname);
225-
exit(1);
226-
}
227-
}
228-
else
229-
{
230-
int fd = dup(fileno(file));
231-
232-
if (fd < 0)
233-
{
234-
pg_log_error("could not duplicate stdout: %m");
235-
exit(1);
236-
}
237-
238-
streamer->gzfile = gzdopen(fd, "wb");
239-
if (streamer->gzfile == NULL)
240-
{
241-
pg_log_error("could not open output file: %m");
242-
exit(1);
243-
}
244-
}
245-
246-
if (gzsetparams(streamer->gzfile, compresslevel,
247-
Z_DEFAULT_STRATEGY) != Z_OK)
248-
{
249-
pg_log_error("could not set compression level %d: %s",
250-
compresslevel, get_gz_error(streamer->gzfile));
251-
exit(1);
252-
}
253-
254-
return &streamer->base;
255-
#else
256-
pg_log_error("this build does not support compression");
257-
exit(1);
258-
#endif
259-
}
260-
261-
#ifdef HAVE_LIBZ
262-
/*
263-
* Write archive content to gzip file.
264-
*/
265-
static void
266-
bbstreamer_gzip_writer_content(bbstreamer *streamer,
267-
bbstreamer_member *member, const char *data,
268-
int len, bbstreamer_archive_context context)
269-
{
270-
bbstreamer_gzip_writer *mystreamer;
271-
272-
mystreamer = (bbstreamer_gzip_writer *) streamer;
273-
274-
if (len == 0)
275-
return;
276-
277-
errno = 0;
278-
if (gzwrite(mystreamer->gzfile, data, len) != len)
279-
{
280-
/* if write didn't set errno, assume problem is no disk space */
281-
if (errno == 0)
282-
errno = ENOSPC;
283-
pg_log_error("could not write to compressed file \"%s\": %s",
284-
mystreamer->pathname, get_gz_error(mystreamer->gzfile));
285-
exit(1);
286-
}
287-
}
288-
289-
/*
290-
* End-of-archive processing when writing to a gzip file consists of just
291-
* calling gzclose.
292-
*
293-
* It makes no difference whether we opened the file or the caller did it,
294-
* because libz provides no way of avoiding a close on the underling file
295-
* handle. Notice, however, that bbstreamer_gzip_writer_new() uses dup() to
296-
* work around this issue, so that the behavior from the caller's viewpoint
297-
* is the same as for bbstreamer_plain_writer.
298-
*/
299-
static void
300-
bbstreamer_gzip_writer_finalize(bbstreamer *streamer)
301-
{
302-
bbstreamer_gzip_writer *mystreamer;
303-
304-
mystreamer = (bbstreamer_gzip_writer *) streamer;
305-
306-
errno = 0; /* in case gzclose() doesn't set it */
307-
if (gzclose(mystreamer->gzfile) != 0)
308-
{
309-
pg_log_error("could not close compressed file \"%s\": %m",
310-
mystreamer->pathname);
311-
exit(1);
312-
}
313-
314-
mystreamer->gzfile = NULL;
315-
}
316-
317-
/*
318-
* Free memory associated with this bbstreamer.
319-
*/
320-
static void
321-
bbstreamer_gzip_writer_free(bbstreamer *streamer)
322-
{
323-
bbstreamer_gzip_writer *mystreamer;
324-
325-
mystreamer = (bbstreamer_gzip_writer *) streamer;
326-
327-
Assert(mystreamer->base.bbs_next == NULL);
328-
Assert(mystreamer->gzfile == NULL);
329-
330-
pfree(mystreamer->pathname);
331-
pfree(mystreamer);
332-
}
333-
334-
/*
335-
* Helper function for libz error reporting.
336-
*/
337-
static const char *
338-
get_gz_error(gzFile gzf)
339-
{
340-
int errnum;
341-
const char *errmsg;
342-
343-
errmsg = gzerror(gzf, &errnum);
344-
if (errnum == Z_ERRNO)
345-
return strerror(errno);
346-
else
347-
return errmsg;
348-
}
349-
#endif
350-
351169
/*
352170
* Create a bbstreamer that extracts an archive.
353171
*

0 commit comments

Comments
 (0)