Skip to content

Commit f081a48

Browse files
tvondratvondra
committed
Unify buffer sizes in pg_dump compression API
Prior to the introduction of the compression API in e996073, pg_dump would use the ZLIB_IN_SIZE/ZLIB_OUT_SIZE to size input/output buffers. Commit 0da243f introduced similar constants for LZ4, but while gzip defined both buffers to be 4kB, LZ4 used 4kB and 16kB without any clear reasoning why that's desirable. Furthermore, parts of the code unaware of which compression is used (e.g. pg_backup_directory.c) continued to use ZLIB_OUT_SIZE directly. Simplify by replacing the various constants with DEFAULT_IO_BUFFER_SIZE, set to 4kB. The compression implementations still have an option to use a custom value, but considering 4kB was fine for 20+ years, I find that unlikely (and we'd probably just increase the default buffer size). Author: Georgios Kokolatos Reviewed-by: Tomas Vondra, Justin Pryzby Discussion: https://postgr.es/m/33496f7c-3449-1426-d568-63f6bca2ac1f@gmail.com
1 parent d3b5775 commit f081a48

File tree

5 files changed

+21
-25
lines changed

5 files changed

+21
-25
lines changed

‎src/bin/pg_dump/compress_gzip.c

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -120,8 +120,8 @@ WriteDataToArchiveGzip(ArchiveHandle *AH, CompressorState *cs,
120120
* actually allocate one extra byte because some routines want to
121121
* append a trailing zero byte to the zlib output.
122122
*/
123-
gzipcs->outbuf = pg_malloc(ZLIB_OUT_SIZE + 1);
124-
gzipcs->outsize = ZLIB_OUT_SIZE;
123+
gzipcs->outsize = DEFAULT_IO_BUFFER_SIZE;
124+
gzipcs->outbuf = pg_malloc(gzipcs->outsize + 1);
125125

126126
/*
127127
* A level of zero simply copies the input one block at the time. This
@@ -158,10 +158,10 @@ ReadDataFromArchiveGzip(ArchiveHandle *AH, CompressorState *cs)
158158
zp->zfree = Z_NULL;
159159
zp->opaque = Z_NULL;
160160

161-
buf = pg_malloc(ZLIB_IN_SIZE);
162-
buflen = ZLIB_IN_SIZE;
161+
buflen = DEFAULT_IO_BUFFER_SIZE;
162+
buf = pg_malloc(buflen);
163163

164-
out = pg_malloc(ZLIB_OUT_SIZE + 1);
164+
out = pg_malloc(DEFAULT_IO_BUFFER_SIZE + 1);
165165

166166
if (inflateInit(zp) != Z_OK)
167167
pg_fatal("could not initialize compression library: %s",
@@ -176,14 +176,14 @@ ReadDataFromArchiveGzip(ArchiveHandle *AH, CompressorState *cs)
176176
while (zp->avail_in > 0)
177177
{
178178
zp->next_out = (void *) out;
179-
zp->avail_out = ZLIB_OUT_SIZE;
179+
zp->avail_out = DEFAULT_IO_BUFFER_SIZE;
180180

181181
res = inflate(zp, 0);
182182
if (res != Z_OK && res != Z_STREAM_END)
183183
pg_fatal("could not uncompress data: %s", zp->msg);
184184

185-
out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
186-
ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
185+
out[DEFAULT_IO_BUFFER_SIZE - zp->avail_out] = '\0';
186+
ahwrite(out, 1, DEFAULT_IO_BUFFER_SIZE - zp->avail_out, AH);
187187
}
188188
}
189189

@@ -192,13 +192,13 @@ ReadDataFromArchiveGzip(ArchiveHandle *AH, CompressorState *cs)
192192
while (res != Z_STREAM_END)
193193
{
194194
zp->next_out = (void *) out;
195-
zp->avail_out = ZLIB_OUT_SIZE;
195+
zp->avail_out = DEFAULT_IO_BUFFER_SIZE;
196196
res = inflate(zp, 0);
197197
if (res != Z_OK && res != Z_STREAM_END)
198198
pg_fatal("could not uncompress data: %s", zp->msg);
199199

200-
out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
201-
ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
200+
out[DEFAULT_IO_BUFFER_SIZE - zp->avail_out] = '\0';
201+
ahwrite(out, 1, DEFAULT_IO_BUFFER_SIZE - zp->avail_out, AH);
202202
}
203203

204204
if (inflateEnd(zp) != Z_OK)

‎src/bin/pg_dump/compress_io.h

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,8 @@
1717

1818
#include "pg_backup_archiver.h"
1919

20-
/* Initial buffer sizes used in zlib compression. */
21-
#define ZLIB_OUT_SIZE 4096
22-
#define ZLIB_IN_SIZE 4096
20+
/* Default size used for IO buffers */
21+
#define DEFAULT_IO_BUFFER_SIZE 4096
2322

2423
extern char *supports_compression(const pg_compress_specification compression_spec);
2524

‎src/bin/pg_dump/compress_lz4.c

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,6 @@
2020
#include <lz4.h>
2121
#include <lz4frame.h>
2222

23-
#define LZ4_OUT_SIZE (4 * 1024)
24-
#define LZ4_IN_SIZE (16 * 1024)
25-
2623
/*
2724
* LZ4F_HEADER_SIZE_MAX first appeared in v1.7.5 of the library.
2825
* Redefine it for installations with a lesser version.
@@ -57,7 +54,7 @@ ReadDataFromArchiveLZ4(ArchiveHandle *AH, CompressorState *cs)
5754
size_t buflen;
5855
size_t cnt;
5956

60-
buflen = LZ4_IN_SIZE;
57+
buflen = DEFAULT_IO_BUFFER_SIZE;
6158
buf = pg_malloc(buflen);
6259
decbuf = pg_malloc(buflen);
6360

@@ -208,7 +205,7 @@ LZ4File_init(LZ4File *fs, int size, bool compressing)
208205

209206
if (fs->compressing)
210207
{
211-
fs->buflen = LZ4F_compressBound(LZ4_IN_SIZE, &fs->prefs);
208+
fs->buflen = LZ4F_compressBound(DEFAULT_IO_BUFFER_SIZE, &fs->prefs);
212209
if (fs->buflen < LZ4F_HEADER_SIZE_MAX)
213210
fs->buflen = LZ4F_HEADER_SIZE_MAX;
214211

@@ -244,7 +241,7 @@ LZ4File_init(LZ4File *fs, int size, bool compressing)
244241
return false;
245242
}
246243

247-
fs->buflen = size > LZ4_OUT_SIZE ? size : LZ4_OUT_SIZE;
244+
fs->buflen = Max(size, DEFAULT_IO_BUFFER_SIZE);
248245
fs->buffer = pg_malloc(fs->buflen);
249246

250247
fs->overflowalloclen = fs->buflen;
@@ -423,7 +420,7 @@ LZ4File_write(const void *ptr, size_t size, CompressFileHandle *CFH)
423420

424421
while (remaining > 0)
425422
{
426-
int chunk = remaining < LZ4_IN_SIZE ? remaining : LZ4_IN_SIZE;
423+
int chunk = Min(remaining, DEFAULT_IO_BUFFER_SIZE);
427424

428425
remaining -= chunk;
429426

‎src/bin/pg_dump/compress_none.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@ ReadDataFromArchiveNone(ArchiveHandle *AH, CompressorState *cs)
3333
char *buf;
3434
size_t buflen;
3535

36-
buf = pg_malloc(ZLIB_OUT_SIZE);
37-
buflen = ZLIB_OUT_SIZE;
36+
buflen = DEFAULT_IO_BUFFER_SIZE;
37+
buf = pg_malloc(buflen);
3838

3939
while ((cnt = cs->readF(AH, &buf, &buflen)))
4040
{

‎src/bin/pg_dump/pg_backup_directory.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -394,8 +394,8 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
394394
if (!CFH)
395395
pg_fatal("could not open input file \"%s\": %m", filename);
396396

397-
buf = pg_malloc(ZLIB_OUT_SIZE);
398-
buflen = ZLIB_OUT_SIZE;
397+
buflen = DEFAULT_IO_BUFFER_SIZE;
398+
buf = pg_malloc(buflen);
399399

400400
while (CFH->read_func(buf, buflen, &cnt, CFH) && cnt > 0)
401401
{

0 commit comments

Comments
 (0)