Skip to content

Commit 70b96af

Browse files
dAdAbirddAdAbird
committed
pg_basebackup: encrypt streamed WAL with new key
Before, pg_basebackup would encrypt streamed WAL according to the keys in pg_tde/wal_keys in the destination dir. This commit introduces the number of changes: pg_basebackup encrypts WAL only if the "-E --encrypt-wal" flag is provided. In such a case, it would extract the principal key, truncate pg_tde/wal_keys and encrypt WAL with a newly generated WAL key. We still expect pg_tde/wal_keys and pg_tde/1664_providers in the destination dir. In case these files are not provided, but "-E" is specified, it fails with an error.
1 parent 167aef2 commit 70b96af

File tree

11 files changed

+165
-27
lines changed

11 files changed

+165
-27
lines changed

‎contrib/pg_tde/meson.build

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ tap_tests = [
109109
't/key_rotate_tablespace.pl',
110110
't/key_validation.pl',
111111
't/multiple_extensions.pl',
112+
't/pg_basebackup.pl',
112113
't/pg_tde_change_key_provider.pl',
113114
't/pg_rewind_basic.pl',
114115
't/pg_rewind_databases.pl',

‎contrib/pg_tde/src/access/pg_tde_xlog_keys.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -715,7 +715,6 @@ pg_tde_save_server_key_redo(const TDESignedPrincipalKeyInfo *signed_key_info)
715715
}
716716
#endif
717717

718-
#ifndef FRONTEND
719718
/*
720719
* Creates the key file and saves the principal key information.
721720
*
@@ -737,17 +736,18 @@ pg_tde_save_server_key(const TDEPrincipalKey *principal_key, bool write_xlog)
737736

738737
pg_tde_sign_principal_key_info(&signed_key_Info, principal_key);
739738

739+
#ifndef FRONTEND
740740
if (write_xlog)
741741
{
742742
XLogBeginInsert();
743743
XLogRegisterData((char *) &signed_key_Info, sizeof(TDESignedPrincipalKeyInfo));
744744
XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_PRINCIPAL_KEY);
745745
}
746+
#endif
746747

747748
fd = pg_tde_open_wal_key_file_write(get_wal_key_file_path(), &signed_key_Info, true, &curr_pos);
748749
CloseTransientFile(fd);
749750
}
750-
#endif
751751

752752
/*
753753
* Get the principal key from the key file. The caller must hold

‎contrib/pg_tde/src/access/pg_tde_xlog_smgr.c

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -326,35 +326,21 @@ TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset,
326326
return pg_pwrite(fd, enc_buff, count, offset);
327327
}
328328

329-
static ssize_t
330-
tdeheap_xlog_seg_write(int fd, const void *buf, size_t count, off_t offset,
331-
TimeLineID tli, XLogSegNo segno, int segSize)
329+
/*
330+
* Set the last (most recent) key's start location if not set.
331+
*/
332+
bool
333+
tde_ensure_xlog_key_location(WalLocation loc)
332334
{
333335
bool lastKeyUsable;
334-
bool afterWriteKey;
335336
#ifdef FRONTEND
336337
bool crashRecovery = false;
337338
#else
338339
bool crashRecovery = GetRecoveryState() == RECOVERY_STATE_CRASH;
339340
#endif
340341

341-
WalLocation loc = {.tli = tli};
342-
WalLocation writeKeyLoc;
343-
344-
XLogSegNoOffsetToRecPtr(segno, offset, segSize, loc.lsn);
345-
346-
/*
347-
* Set the last (most recent) key's start LSN if not set.
348-
*
349-
* This func called with WALWriteLock held, so no need in any extra sync.
350-
*/
351-
352-
writeKeyLoc.lsn = TDEXLogGetEncKeyLsn();
342+
lastKeyUsable = (TDEXLogGetEncKeyLsn() != 0);
353343
pg_read_barrier();
354-
writeKeyLoc.tli = TDEXLogGetEncKeyTli();
355-
356-
lastKeyUsable = (writeKeyLoc.lsn != 0);
357-
afterWriteKey = wal_location_cmp(writeKeyLoc, loc) <= 0;
358344

359345
if (EncryptionKey.type != WAL_KEY_TYPE_INVALID && !lastKeyUsable)
360346
{
@@ -372,6 +358,30 @@ tdeheap_xlog_seg_write(int fd, const void *buf, size_t count, off_t offset,
372358
}
373359
}
374360

361+
return lastKeyUsable;
362+
}
363+
364+
static ssize_t
365+
tdeheap_xlog_seg_write(int fd, const void *buf, size_t count, off_t offset,
366+
TimeLineID tli, XLogSegNo segno, int segSize)
367+
{
368+
bool lastKeyUsable;
369+
bool afterWriteKey;
370+
WalLocation writeKeyLoc;
371+
WalLocation loc = {.tli = tli};
372+
373+
XLogSegNoOffsetToRecPtr(segno, offset, segSize, loc.lsn);
374+
lastKeyUsable = tde_ensure_xlog_key_location(loc);
375+
376+
/*
377+
* On backend this called with WALWriteLock held, so no need in any extra
378+
* sync.
379+
*/
380+
writeKeyLoc.lsn = TDEXLogGetEncKeyLsn();
381+
pg_read_barrier();
382+
writeKeyLoc.tli = TDEXLogGetEncKeyTli();
383+
afterWriteKey = wal_location_cmp(writeKeyLoc, loc) <= 0;
384+
375385
if ((!afterWriteKey || !lastKeyUsable) && EncryptionKey.type != WAL_KEY_TYPE_INVALID)
376386
{
377387
return TDEXLogWriteEncryptedPagesOldKeys(fd, buf, count, offset, tli, segno, segSize);

‎contrib/pg_tde/src/include/access/pg_tde_xlog_smgr.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77

88
#include "postgres.h"
99

10+
#include "access/pg_tde_xlog_keys.h"
11+
1012
extern Size TDEXLogEncryptStateSize(void);
1113
extern void TDEXLogShmemInit(void);
1214
extern void TDEXLogSmgrInit(void);
@@ -16,4 +18,6 @@ extern void TDEXLogSmgrInitWriteReuseKey(void);
1618
extern void TDEXLogCryptBuffer(const void *buf, void *out_buf, size_t count, off_t offset,
1719
TimeLineID tli, XLogSegNo segno, int segSize);
1820

21+
extern bool tde_ensure_xlog_key_location(WalLocation loc);
22+
1923
#endif /* PG_TDE_XLOGSMGR_H */

‎contrib/pg_tde/t/pg_basebackup.pl

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
2+
# Copyright (c) 2021-2024, PostgreSQL Global Development Group
3+
4+
use strict;
5+
use warnings FATAL => 'all';
6+
use Config;
7+
use File::Basename qw(basename dirname);
8+
use File::Path qw(rmtree);
9+
use PostgreSQL::Test::Cluster;
10+
use PostgreSQL::Test::Utils;
11+
use Test::More;
12+
13+
program_help_ok('pg_basebackup');
14+
program_version_ok('pg_basebackup');
15+
program_options_handling_ok('pg_basebackup');
16+
17+
my $tempdir = PostgreSQL::Test::Utils::tempdir;
18+
19+
my $node = PostgreSQL::Test::Cluster->new('main');
20+
21+
# Initialize node without replication settings
22+
$node->init(
23+
allows_streaming => 1,
24+
extra => ['--data-checksums'],
25+
auth_extra => [ '--create-role', 'backupuser' ]);
26+
$node->start;
27+
28+
# Sanity checks for options with WAL encryption
29+
$node->command_fails_like(
30+
[ 'pg_basebackup', '-D', "$tempdir/backup", '-E', '-Ft' ],
31+
qr/can not encrypt WAL in tar mode/,
32+
'encryption in tar mode');
33+
34+
$node->command_fails_like(
35+
[ 'pg_basebackup', '-D', "$tempdir/backup", '-E', '-X', 'fetch' ],
36+
qr/WAL encryption can only be used with WAL streaming/,
37+
'encryption with WAL fetch');
38+
39+
$node->command_fails_like(
40+
[ 'pg_basebackup', '-D', "$tempdir/backup", '-E', '-X', 'none' ],
41+
qr/WAL encryption can only be used with WAL streaming/,
42+
'encryption with WAL none');
43+
44+
$node->command_fails_like(
45+
[ 'pg_basebackup', '-D', "$tempdir/backup", '-E' ],
46+
qr/could not find global principal key/,
47+
'encryption with no pg_tde dir');
48+
49+
done_testing();

‎contrib/pg_tde/t/pgtde.pm

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,8 @@ sub backup
119119
PostgreSQL::Test::RecursiveCopy::copypath($node->data_dir . '/pg_tde',
120120
$backup_dir . '/pg_tde');
121121

122+
push @{ $params{backup_options} }, "-E";
123+
122124
$node->backup($backup_name, %params);
123125
}
124126

‎src/bin/pg_basebackup/bbstreamer.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,8 @@ extern bbstreamer *bbstreamer_gzip_writer_new(char *pathname, FILE *file,
204204
pg_compress_specification *compress);
205205
extern bbstreamer *bbstreamer_extractor_new(const char *basepath,
206206
const char *(*link_map) (const char *),
207-
void (*report_output_file) (const char *));
207+
void (*report_output_file) (const char *),
208+
bool encrypted_wal);
208209

209210
extern bbstreamer *bbstreamer_gzip_decompressor_new(bbstreamer *next);
210211
extern bbstreamer *bbstreamer_lz4_compressor_new(bbstreamer *next,

‎src/bin/pg_basebackup/bbstreamer_file.c

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ typedef struct bbstreamer_extractor
3838
void (*report_output_file) (const char *);
3939
char filename[MAXPGPATH];
4040
FILE *file;
41+
bool encryped_wal;
4142
} bbstreamer_extractor;
4243

4344
static void bbstreamer_plain_writer_content(bbstreamer *streamer,
@@ -186,7 +187,8 @@ bbstreamer_plain_writer_free(bbstreamer *streamer)
186187
bbstreamer *
187188
bbstreamer_extractor_new(const char *basepath,
188189
const char *(*link_map) (const char *),
189-
void (*report_output_file) (const char *))
190+
void (*report_output_file) (const char *),
191+
bool encrypted_wal)
190192
{
191193
bbstreamer_extractor *streamer;
192194

@@ -196,6 +198,7 @@ bbstreamer_extractor_new(const char *basepath,
196198
streamer->basepath = pstrdup(basepath);
197199
streamer->link_map = link_map;
198200
streamer->report_output_file = report_output_file;
201+
streamer->encryped_wal = encrypted_wal;
199202

200203
return &streamer->base;
201204
}
@@ -240,9 +243,21 @@ bbstreamer_extractor_content(bbstreamer *streamer, bbstreamer_member *member,
240243
extract_link(mystreamer->filename, linktarget);
241244
}
242245
else
246+
{
247+
#ifdef PERCONA_EXT
248+
/*
249+
* A streamed WAL is encrypted with the newly generated WAL key,
250+
* hence we have to prevent these files from rewriting.
251+
*/
252+
if (mystreamer->encryped_wal &&
253+
(strcmp(member->pathname, "pg_tde/wal_keys") == 0 ||
254+
strcmp(member->pathname, "pg_tde/1664_providers") == 0))
255+
break;
256+
#endif
243257
mystreamer->file =
244258
create_file_for_extract(mystreamer->filename,
245259
member->mode);
260+
}
246261

247262
/* Report output file change. */
248263
if (mystreamer->report_output_file)

‎src/bin/pg_basebackup/pg_basebackup.c

Lines changed: 52 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,12 @@
4141
#ifdef PERCONA_EXT
4242
#include "access/pg_tde_fe_init.h"
4343
#include "access/pg_tde_xlog_smgr.h"
44+
#include "access/pg_tde_xlog_keys.h"
4445
#include "access/xlog_smgr.h"
46+
#include "catalog/tde_principal_key.h"
4547
#include "pg_tde.h"
48+
49+
#define GLOBAL_DATA_TDE_OID 1664
4650
#endif
4751

4852
#define ERRCODE_DATA_CORRUPTED_BCP "XX001"
@@ -145,6 +149,7 @@ static bool showprogress = false;
145149
static bool estimatesize = true;
146150
static int verbose = 0;
147151
static IncludeWal includewal = STREAM_WAL;
152+
static bool encrypt_wal = false;
148153
static bool fastcheckpoint = false;
149154
static bool writerecoveryconf = false;
150155
static bool do_sync = true;
@@ -416,6 +421,9 @@ usage(void)
416421
printf(_(" --waldir=WALDIR location for the write-ahead log directory\n"));
417422
printf(_(" -X, --wal-method=none|fetch|stream\n"
418423
" include required WAL files with specified method\n"));
424+
#ifdef PERCONA_EXT
425+
printf(_(" -E, --encrypt-wal encrypt streamed WAL\n"));
426+
#endif
419427
printf(_(" -z, --gzip compress tar output\n"));
420428
printf(_(" -Z, --compress=[{client|server}-]METHOD[:DETAIL]\n"
421429
" compress on client or server as specified\n"));
@@ -568,6 +576,7 @@ LogStreamerMain(logstreamer_param *param)
568576
stream.synchronous = false;
569577
/* fsync happens at the end of pg_basebackup for all data */
570578
stream.do_sync = false;
579+
stream.encrypt = encrypt_wal;
571580
stream.mark_done = true;
572581
stream.partial_suffix = NULL;
573582
stream.replication_slot = replication_slot;
@@ -662,12 +671,23 @@ StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier,
662671
"pg_xlog" : "pg_wal");
663672

664673
#ifdef PERCONA_EXT
665-
{
674+
if (encrypt_wal) {
666675
char tdedir[MAXPGPATH];
676+
TDEPrincipalKey *principalKey;
667677

668678
snprintf(tdedir, sizeof(tdedir), "%s/%s", basedir, PG_TDE_DATA_DIR);
669679
pg_tde_fe_init(tdedir);
670680
TDEXLogSmgrInit();
681+
682+
principalKey = GetPrincipalKey(GLOBAL_DATA_TDE_OID, NULL);
683+
if (!principalKey)
684+
{
685+
pg_log_error("could not find global principal key");
686+
pg_log_error_hint("Copy PGDATA/pg_tde from the source to the backup destination dir.");
687+
exit(1);
688+
}
689+
pg_tde_save_server_key(principalKey, false);
690+
TDEXLogSmgrInitWrite(true);
671691
}
672692
#endif
673693

@@ -1187,7 +1207,8 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
11871207
directory = get_tablespace_mapping(spclocation);
11881208
streamer = bbstreamer_extractor_new(directory,
11891209
get_tablespace_mapping,
1190-
progress_update_filename);
1210+
progress_update_filename,
1211+
encrypt_wal);
11911212
}
11921213
else
11931214
{
@@ -2393,6 +2414,9 @@ main(int argc, char **argv)
23932414
{"target", required_argument, NULL, 't'},
23942415
{"tablespace-mapping", required_argument, NULL, 'T'},
23952416
{"wal-method", required_argument, NULL, 'X'},
2417+
#ifdef PERCONA_EXT
2418+
{"encrypt-wal", no_argument, NULL, 'e'},
2419+
#endif
23962420
{"gzip", no_argument, NULL, 'z'},
23972421
{"compress", required_argument, NULL, 'Z'},
23982422
{"label", required_argument, NULL, 'l'},
@@ -2447,7 +2471,7 @@ main(int argc, char **argv)
24472471

24482472
atexit(cleanup_directories_atexit);
24492473

2450-
while ((c = getopt_long(argc, argv, "c:Cd:D:F:h:i:l:nNp:Pr:Rs:S:t:T:U:vwWX:zZ:",
2474+
while ((c = getopt_long(argc, argv, "c:Cd:D:EF:h:i:l:nNp:Pr:Rs:S:t:T:U:vwWX:zZ:",
24512475
long_options, &option_index)) != -1)
24522476
{
24532477
switch (c)
@@ -2560,6 +2584,11 @@ main(int argc, char **argv)
25602584
pg_fatal("invalid wal-method option \"%s\", must be \"fetch\", \"stream\", or \"none\"",
25612585
optarg);
25622586
break;
2587+
#ifdef PERCONA_EXT
2588+
case 'E':
2589+
encrypt_wal = true;
2590+
break;
2591+
#endif
25632592
case 'z':
25642593
compression_algorithm = "gzip";
25652594
compression_detail = NULL;
@@ -2738,6 +2767,26 @@ main(int argc, char **argv)
27382767
exit(1);
27392768
}
27402769

2770+
/*
2771+
* Sanity checks for WAL encryption.
2772+
*/
2773+
if (encrypt_wal)
2774+
{
2775+
if (includewal != STREAM_WAL)
2776+
{
2777+
pg_log_error("WAL encryption can only be used with WAL streaming");
2778+
pg_log_error_hint("Use -X stream with -E.");
2779+
exit(1);
2780+
}
2781+
2782+
if (format != 'p')
2783+
{
2784+
pg_log_error("can not encrypt WAL in tar mode");
2785+
pg_log_error_hint("Use -Fp with -E.");
2786+
exit(1);
2787+
}
2788+
}
2789+
27412790
/*
27422791
* Sanity checks for replication slot options.
27432792
*/

‎src/bin/pg_basebackup/receivelog.c

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
#ifdef PERCONA_EXT
2929
#include "access/pg_tde_fe_init.h"
3030
#include "access/pg_tde_xlog_smgr.h"
31+
#include "access/xlog_smgr.h"
3132
#include "catalog/tde_global_space.h"
3233
#endif
3334

@@ -1131,8 +1132,13 @@ ProcessXLogDataMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len,
11311132
}
11321133

11331134
#ifdef PERCONA_EXT
1135+
if (stream->encrypt)
11341136
{
11351137
void* enc_buf = copybuf + hdr_len + bytes_written;
1138+
WalLocation loc = {.tli = stream->timeline};
1139+
1140+
XLogSegNoOffsetToRecPtr(segno, xlogoff, WalSegSz, loc.lsn);
1141+
tde_ensure_xlog_key_location(loc);
11361142
TDEXLogCryptBuffer(enc_buf, enc_buf, bytes_to_write,
11371143
xlogoff, stream->timeline, segno, WalSegSz);
11381144
}

0 commit comments

Comments
 (0)