Skip to content

Commit 0da243f

Browse files
tvondratvondra
committed
Add LZ4 compression to pg_dump
Expand pg_dump's compression streaming and file APIs to support the lz4 algorithm. The newly added compress_lz4.{c,h} files cover all the functionality of the aforementioned APIs. Minor changes were necessary in various pg_backup_* files, where code for the 'lz4' file suffix has been added, as well as pg_dump's compression option parsing. Author: Georgios Kokolatos Reviewed-by: Michael Paquier, Rachel Heaton, Justin Pryzby, Shi Yu, Tomas Vondra Discussion: https://postgr.es/m/faUNEOpts9vunEaLnmxmG-DldLSg_ql137OC3JYDmgrOMHm1RvvWY2IdBkv_CRxm5spCCb_OmKNk2T03TMm0fBEWveFF9wA1WizPuAgB7Ss%3D%40protonmail.com
1 parent e0b3074 commit 0da243f

File tree

12 files changed

+782
-22
lines changed

12 files changed

+782
-22
lines changed

‎doc/src/sgml/ref/pg_dump.sgml

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -330,9 +330,10 @@ PostgreSQL documentation
330330
machine-readable format that <application>pg_restore</application>
331331
can read. A directory format archive can be manipulated with
332332
standard Unix tools; for example, files in an uncompressed archive
333-
can be compressed with the <application>gzip</application> tool.
334-
This format is compressed by default and also supports parallel
335-
dumps.
333+
can be compressed with the <application>gzip</application> or
334+
<application>lz4</application>tool.
335+
This format is compressed by default using <literal>gzip</literal>
336+
and also supports parallel dumps.
336337
</para>
337338
</listitem>
338339
</varlistentry>
@@ -654,7 +655,7 @@ PostgreSQL documentation
654655
<para>
655656
Specify the compression method and/or the compression level to use.
656657
The compression method can be set to <literal>gzip</literal> or
657-
<literal>none</literal> for no compression.
658+
<literal>lz4</literal> or <literal>none</literal> for no compression.
658659
A compression detail string can optionally be specified. If the
659660
detail string is an integer, it specifies the compression level.
660661
Otherwise, it should be a comma-separated list of items, each of the
@@ -675,8 +676,8 @@ PostgreSQL documentation
675676
individual table-data segments, and the default is to compress using
676677
<literal>gzip</literal> at a moderate level. For plain text output,
677678
setting a nonzero compression level causes the entire output file to be compressed,
678-
as though it had been fed through <application>gzip</application>; but the default
679-
is not to compress.
679+
as though it had been fed through <application>gzip</application> or
680+
<application>lz4</application>; but the default is not to compress.
680681
</para>
681682
<para>
682683
The tar archive format currently does not support compression at all.

‎src/bin/pg_dump/Makefile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ top_builddir = ../../..
1717
include $(top_builddir)/src/Makefile.global
1818

1919
export GZIP_PROGRAM=$(GZIP)
20+
export LZ4
2021
export with_icu
2122

2223
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
@@ -26,6 +27,7 @@ OBJS = \
2627
$(WIN32RES) \
2728
compress_gzip.o \
2829
compress_io.o \
30+
compress_lz4.o \
2931
compress_none.o \
3032
dumputils.o \
3133
parallel.o \

‎src/bin/pg_dump/compress_io.c

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@
5353
* InitDiscoverCompressFileHandle tries to infer the compression by the
5454
* filename suffix. If the suffix is not yet known then it tries to simply
5555
* open the file and if it fails, it tries to open the same file with the .gz
56-
* suffix.
56+
* suffix, and then again with the .lz4 suffix.
5757
*
5858
* IDENTIFICATION
5959
* src/bin/pg_dump/compress_io.c
@@ -67,6 +67,7 @@
6767

6868
#include "compress_gzip.h"
6969
#include "compress_io.h"
70+
#include "compress_lz4.h"
7071
#include "compress_none.h"
7172
#include "pg_backup_utils.h"
7273

@@ -93,6 +94,10 @@ supports_compression(const pg_compress_specification compression_spec)
9394
if (algorithm == PG_COMPRESSION_GZIP)
9495
supported = true;
9596
#endif
97+
#ifdef USE_LZ4
98+
if (algorithm == PG_COMPRESSION_LZ4)
99+
supported = true;
100+
#endif
96101

97102
if (!supported)
98103
return psprintf("this build does not support compression with %s",
@@ -123,6 +128,8 @@ AllocateCompressor(const pg_compress_specification compression_spec,
123128
InitCompressorNone(cs, compression_spec);
124129
else if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
125130
InitCompressorGzip(cs, compression_spec);
131+
else if (compression_spec.algorithm == PG_COMPRESSION_LZ4)
132+
InitCompressorLZ4(cs, compression_spec);
126133

127134
return cs;
128135
}
@@ -187,6 +194,8 @@ InitCompressFileHandle(const pg_compress_specification compression_spec)
187194
InitCompressFileHandleNone(CFH, compression_spec);
188195
else if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
189196
InitCompressFileHandleGzip(CFH, compression_spec);
197+
else if (compression_spec.algorithm == PG_COMPRESSION_LZ4)
198+
InitCompressFileHandleLZ4(CFH, compression_spec);
190199

191200
return CFH;
192201
}
@@ -196,11 +205,11 @@ InitCompressFileHandle(const pg_compress_specification compression_spec)
196205
* be either "r" or "rb".
197206
*
198207
* If the file at 'path' contains the suffix of a supported compression method,
199-
* currently this includes only ".gz", then this compression will be used
208+
* currently this includes ".gz" and ".lz4", then this compression will be used
200209
* throughout. Otherwise the compression will be inferred by iteratively trying
201210
* to open the file at 'path', first as is, then by appending known compression
202211
* suffixes. So if you pass "foo" as 'path', this will open either "foo" or
203-
* "foo.gz", trying in that order.
212+
* "foo.gz" or "foo.lz4", trying in that order.
204213
*
205214
* On failure, return NULL with an error code in errno.
206215
*/
@@ -238,6 +247,17 @@ InitDiscoverCompressFileHandle(const char *path, const char *mode)
238247
if (exists)
239248
compression_spec.algorithm = PG_COMPRESSION_GZIP;
240249
}
250+
#endif
251+
#ifdef USE_LZ4
252+
if (!exists)
253+
{
254+
free_keep_errno(fname);
255+
fname = psprintf("%s.lz4", path);
256+
exists = (stat(fname, &st) == 0);
257+
258+
if (exists)
259+
compression_spec.algorithm = PG_COMPRESSION_LZ4;
260+
}
241261
#endif
242262
}
243263

0 commit comments

Comments
 (0)