diff --git a/contrib/pg_tde/Makefile b/contrib/pg_tde/Makefile index 25726c461125e..02b86b9f8f66c 100644 --- a/contrib/pg_tde/Makefile +++ b/contrib/pg_tde/Makefile @@ -1,7 +1,7 @@ PGFILEDESC = "pg_tde access method" MODULE_big = pg_tde EXTENSION = pg_tde -DATA = pg_tde--1.0.sql +DATA = pg_tde--1.0.sql pg_tde--1.0--2.0.sql # Since meson supports skipping test suites this is a make only feature ifndef TDE_MODE diff --git a/contrib/pg_tde/documentation/docs/functions.md b/contrib/pg_tde/documentation/docs/functions.md index b290d440ee7b7..58f8294e84407 100644 --- a/contrib/pg_tde/documentation/docs/functions.md +++ b/contrib/pg_tde/documentation/docs/functions.md @@ -421,3 +421,22 @@ If any of the above checks fail, the function reports an error. ```sql SELECT pg_tde_verify_default_key(); ``` + +### pg_tde_is_wal_record_encrypted + +This function checks if a WAL record is encrypted or not. It assumes provided LSN belongs to the current timeline if not specified. + +```sql +SELECT pg_tde_is_wal_record_encrypted( + 'wal_record_lsn', + 'optional_timeline_id' +); +``` + +### pg_tde_get_wal_encryption_ranges + +This function returns the ranges of WAL records that are encrypted. It returns a set of records with the start and end LSNs and TLI for each range. + +```sql +SELECT * FROM pg_tde_get_wal_encryption_ranges(); +``` diff --git a/contrib/pg_tde/expected/version.out b/contrib/pg_tde/expected/version.out index 44ebea8d04d32..54e00d133bf6a 100644 --- a/contrib/pg_tde/expected/version.out +++ b/contrib/pg_tde/expected/version.out @@ -2,7 +2,7 @@ CREATE EXTENSION pg_tde; SELECT pg_tde_version(); pg_tde_version ---------------- - pg_tde 1.0.0 + pg_tde 2.0.0 (1 row) DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/meson.build b/contrib/pg_tde/meson.build index 0494fda2796e5..3dfeb332aef85 100644 --- a/contrib/pg_tde/meson.build +++ b/contrib/pg_tde/meson.build @@ -78,6 +78,7 @@ endif install_data( 'pg_tde.control', 'pg_tde--1.0.sql', + 'pg_tde--1.0--2.0.sql', kwargs: contrib_data_args, ) @@ -128,6 +129,7 @@ tap_tests = [ 't/unlogged_tables.pl', 't/wal_archiving.pl', 't/wal_encrypt.pl', + 't/wal_encryption_ranges.pl', 't/wal_key_tli.pl', 't/2pc_replication.pl', 't/stream_rep.pl', diff --git a/contrib/pg_tde/pg_tde--1.0--2.0.sql b/contrib/pg_tde/pg_tde--1.0--2.0.sql new file mode 100644 index 0000000000000..2596ec8c5f939 --- /dev/null +++ b/contrib/pg_tde/pg_tde--1.0--2.0.sql @@ -0,0 +1,17 @@ +-- Function to check if a WAL record is encrypted +CREATE FUNCTION pg_tde_is_wal_record_encrypted(lsn pg_lsn, tli integer DEFAULT 0) +RETURNS BOOLEAN +LANGUAGE C +AS 'MODULE_PATHNAME'; +REVOKE ALL ON FUNCTION pg_tde_is_wal_record_encrypted(pg_lsn, integer) FROM PUBLIC; + +-- Function to get WAL encryption ranges +CREATE FUNCTION pg_tde_get_wal_encryption_ranges + (OUT start_tli integer, + OUT start_lsn pg_lsn, + OUT end_tli integer, + OUT end_lsn pg_lsn) +RETURNS SETOF RECORD +LANGUAGE C +AS 'MODULE_PATHNAME'; +REVOKE ALL ON FUNCTION pg_tde_get_wal_encryption_ranges() FROM PUBLIC; diff --git a/contrib/pg_tde/pg_tde.control b/contrib/pg_tde/pg_tde.control index 9ea82992d7490..5496ed190ecc4 100644 --- a/contrib/pg_tde/pg_tde.control +++ b/contrib/pg_tde/pg_tde.control @@ -1,4 +1,4 @@ comment = 'pg_tde access method' -default_version = '1.0' +default_version = '2.0' module_pathname = '$libdir/pg_tde' relocatable = false diff --git a/contrib/pg_tde/src/include/pg_tde.h b/contrib/pg_tde/src/include/pg_tde.h index 4b6bb94d6d8f4..46e622d26a91c 100644 --- a/contrib/pg_tde/src/include/pg_tde.h +++ b/contrib/pg_tde/src/include/pg_tde.h @@ -2,7 +2,7 @@ #define PG_TDE_H #define PG_TDE_NAME "pg_tde" -#define PG_TDE_VERSION "1.0.0" +#define PG_TDE_VERSION "2.0.0" #define PG_TDE_VERSION_STRING PG_TDE_NAME " " PG_TDE_VERSION #define PG_TDE_DATA_DIR "pg_tde" diff --git a/contrib/pg_tde/src/pg_tde.c b/contrib/pg_tde/src/pg_tde.c index e6f32f099bbe0..7857c8b740d63 100644 --- a/contrib/pg_tde/src/pg_tde.c +++ b/contrib/pg_tde/src/pg_tde.c @@ -15,10 +15,12 @@ #include "storage/shmem.h" #include "utils/builtins.h" #include "utils/percona.h" +#include "utils/pg_lsn.h" #include "access/pg_tde_tdemap.h" #include "access/pg_tde_xlog.h" #include "access/pg_tde_xlog_smgr.h" +#include "access/pg_tde_xlog_keys.h" #include "catalog/tde_global_space.h" #include "catalog/tde_principal_key.h" #include "encryption/enc_aes.h" @@ -33,6 +35,8 @@ PG_MODULE_MAGIC; +#define PG_TDE_LIST_WAL_KEYS_RANGES_COLS 4 + static void pg_tde_init_data_dir(void); static shmem_startup_hook_type prev_shmem_startup_hook = NULL; @@ -41,6 +45,8 @@ static shmem_request_hook_type prev_shmem_request_hook = NULL; PG_FUNCTION_INFO_V1(pg_tde_extension_initialize); PG_FUNCTION_INFO_V1(pg_tde_version); PG_FUNCTION_INFO_V1(pg_tdeam_handler); +PG_FUNCTION_INFO_V1(pg_tde_is_wal_record_encrypted); +PG_FUNCTION_INFO_V1(pg_tde_get_wal_encryption_ranges); static void tde_shmem_request(void) @@ -166,3 +172,100 @@ pg_tdeam_handler(PG_FUNCTION_ARGS) { PG_RETURN_POINTER(GetHeapamTableAmRoutine()); } + +/* + * Returns true if the WAL record at the given LSN is encrypted. + */ +Datum +pg_tde_is_wal_record_encrypted(PG_FUNCTION_ARGS) +{ + XLogRecPtr lsn = PG_GETARG_LSN(0); + int tli = PG_GETARG_INT32(1); + WalLocation loc; + WALKeyCacheRec *keys; + + if (tli == 0) + tli = GetWALInsertionTimeLine(); + + /* Load all keys for the given timeline */ + loc = (WalLocation) + { + .tli = tli,.lsn = 0 + }; + + keys = pg_tde_fetch_wal_keys(loc); + if (!keys) + PG_RETURN_BOOL(false); + + loc.lsn = lsn; + + for (WALKeyCacheRec *curr_key = keys; curr_key != NULL; curr_key = curr_key->next) + { + if (wal_location_cmp(loc, curr_key->start) >= 0 && + wal_location_cmp(loc, curr_key->end) < 0) + PG_RETURN_BOOL(curr_key->key.type == WAL_KEY_TYPE_ENCRYPTED); + } + + PG_RETURN_BOOL(false); +} + +/* + * Returns WAL encryption ranges. WAL records within the LSN range are encrypted. + */ +Datum +pg_tde_get_wal_encryption_ranges(PG_FUNCTION_ARGS) +{ + Tuplestorestate *tupstore; + TupleDesc tupdesc; + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + MemoryContext per_query_ctx; + MemoryContext oldcontext; + WALKeyCacheRec *keys; + WalLocation loc = {.tli = 0,.lsn = 0}; + + /* check to see if caller supports us returning a tuplestore */ + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("set-valued function called in context that cannot accept a set")); + if (!(rsinfo->allowedModes & SFRM_Materialize)) + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("materialize mode required, but it is not allowed in this context")); + + /* Switch into long-lived context to construct returned data structures */ + per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; + oldcontext = MemoryContextSwitchTo(per_query_ctx); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + tupstore = tuplestore_begin_heap(true, false, work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; + + MemoryContextSwitchTo(oldcontext); + + keys = pg_tde_fetch_wal_keys(loc); + + for (WALKeyCacheRec *curr_key = keys; curr_key != NULL; curr_key = curr_key->next) + { + Datum values[PG_TDE_LIST_WAL_KEYS_RANGES_COLS] = {0}; + bool nulls[PG_TDE_LIST_WAL_KEYS_RANGES_COLS] = {0}; + int i = 0; + + if (curr_key->key.type != WAL_KEY_TYPE_ENCRYPTED) + continue; + + values[i++] = Int64GetDatum(curr_key->start.tli); + values[i++] = Int64GetDatum(curr_key->start.lsn); + values[i++] = Int64GetDatum(curr_key->end.tli); + values[i++] = Int64GetDatum(curr_key->end.lsn); + + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } + + return (Datum) 0; +} diff --git a/contrib/pg_tde/t/expected/basic.out b/contrib/pg_tde/t/expected/basic.out index 3947c0988f891..abeba5d06e232 100644 --- a/contrib/pg_tde/t/expected/basic.out +++ b/contrib/pg_tde/t/expected/basic.out @@ -20,7 +20,7 @@ CREATE EXTENSION pg_tde; SELECT extname, extversion FROM pg_extension WHERE extname = 'pg_tde'; extname | extversion ---------+------------ - pg_tde | 1.0 + pg_tde | 2.0 (1 row) CREATE TABLE test_enc (id SERIAL, k INTEGER, PRIMARY KEY (id)) USING tde_heap; diff --git a/contrib/pg_tde/t/expected/wal_encrypt.out b/contrib/pg_tde/t/expected/wal_encrypt.out index 5f374e9fbb9a2..daeb07320b9d7 100644 --- a/contrib/pg_tde/t/expected/wal_encrypt.out +++ b/contrib/pg_tde/t/expected/wal_encrypt.out @@ -13,6 +13,12 @@ SELECT key_name, provider_name, provider_id FROM pg_tde_server_key_info(); | | (1 row) +SELECT pg_tde_is_wal_record_encrypted(pg_current_wal_lsn()); + pg_tde_is_wal_record_encrypted +-------------------------------- + f +(1 row) + SELECT pg_tde_create_key_using_global_key_provider('server-key', 'file-keyring-010'); pg_tde_create_key_using_global_key_provider --------------------------------------------- @@ -53,6 +59,12 @@ SELECT slot_name FROM pg_create_logical_replication_slot('tde_slot', 'test_decod CREATE TABLE test_wal (id SERIAL, k INTEGER, PRIMARY KEY (id)); INSERT INTO test_wal (k) VALUES (1), (2); +SELECT pg_tde_is_wal_record_encrypted(pg_current_wal_lsn()); + pg_tde_is_wal_record_encrypted +-------------------------------- + t +(1 row) + ALTER SYSTEM SET pg_tde.wal_encrypt = off; -- server restart without wal encryption SHOW pg_tde.wal_encrypt; @@ -62,6 +74,12 @@ SHOW pg_tde.wal_encrypt; (1 row) INSERT INTO test_wal (k) VALUES (3), (4); +SELECT pg_tde_is_wal_record_encrypted(pg_current_wal_lsn()); + pg_tde_is_wal_record_encrypted +-------------------------------- + f +(1 row) + ALTER SYSTEM SET pg_tde.wal_encrypt = on; -- server restart with wal encryption SHOW pg_tde.wal_encrypt; @@ -71,6 +89,12 @@ SHOW pg_tde.wal_encrypt; (1 row) INSERT INTO test_wal (k) VALUES (5), (6); +SELECT pg_tde_is_wal_record_encrypted(pg_current_wal_lsn()); + pg_tde_is_wal_record_encrypted +-------------------------------- + t +(1 row) + -- server restart with still wal encryption SHOW pg_tde.wal_encrypt; pg_tde.wal_encrypt @@ -79,6 +103,12 @@ SHOW pg_tde.wal_encrypt; (1 row) INSERT INTO test_wal (k) VALUES (7), (8); +SELECT pg_tde_is_wal_record_encrypted(pg_current_wal_lsn()); + pg_tde_is_wal_record_encrypted +-------------------------------- + t +(1 row) + SELECT data FROM pg_logical_slot_get_changes('tde_slot', NULL, NULL); data ----------------------------------------------------------- diff --git a/contrib/pg_tde/t/wal_encrypt.pl b/contrib/pg_tde/t/wal_encrypt.pl index 2c23d75808a52..10554f44c01c0 100644 --- a/contrib/pg_tde/t/wal_encrypt.pl +++ b/contrib/pg_tde/t/wal_encrypt.pl @@ -31,6 +31,9 @@ 'SELECT key_name, provider_name, provider_id FROM pg_tde_server_key_info();' ); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_is_wal_record_encrypted(pg_current_wal_lsn());"); + PGTDE::psql($node, 'postgres', "SELECT pg_tde_create_key_using_global_key_provider('server-key', 'file-keyring-010');" ); @@ -60,6 +63,9 @@ PGTDE::psql($node, 'postgres', 'INSERT INTO test_wal (k) VALUES (1), (2);'); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_is_wal_record_encrypted(pg_current_wal_lsn());"); + PGTDE::psql($node, 'postgres', 'ALTER SYSTEM SET pg_tde.wal_encrypt = off;'); PGTDE::append_to_result_file("-- server restart without wal encryption"); @@ -69,6 +75,9 @@ PGTDE::psql($node, 'postgres', 'INSERT INTO test_wal (k) VALUES (3), (4);'); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_is_wal_record_encrypted(pg_current_wal_lsn());"); + PGTDE::psql($node, 'postgres', 'ALTER SYSTEM SET pg_tde.wal_encrypt = on;'); PGTDE::append_to_result_file("-- server restart with wal encryption"); @@ -78,6 +87,9 @@ PGTDE::psql($node, 'postgres', 'INSERT INTO test_wal (k) VALUES (5), (6);'); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_is_wal_record_encrypted(pg_current_wal_lsn());"); + PGTDE::append_to_result_file("-- server restart with still wal encryption"); $node->restart; @@ -85,6 +97,9 @@ PGTDE::psql($node, 'postgres', 'INSERT INTO test_wal (k) VALUES (7), (8);'); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_is_wal_record_encrypted(pg_current_wal_lsn());"); + PGTDE::psql($node, 'postgres', "SELECT data FROM pg_logical_slot_get_changes('tde_slot', NULL, NULL);"); diff --git a/contrib/pg_tde/t/wal_encryption_ranges.pl b/contrib/pg_tde/t/wal_encryption_ranges.pl new file mode 100644 index 0000000000000..e0c1bd433179b --- /dev/null +++ b/contrib/pg_tde/t/wal_encryption_ranges.pl @@ -0,0 +1,95 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use File::Basename; +use Test::More; +use lib 't'; +use pgtde; + +PGTDE::setup_files_dir(basename($0)); + +unlink('/tmp/wal_encryption_ranges.per'); + +my $psql_out = ''; + +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf('postgresql.conf', "shared_preload_libraries = 'pg_tde'"); +$node->append_conf('postgresql.conf', "wal_level = 'logical'"); +$node->start; + +# Create and configure pg_tde extension +$node->psql('postgres', "CREATE EXTENSION pg_tde;"); + +$node->psql('postgres', + "SELECT pg_tde_add_global_key_provider_file('file-keyring', '/tmp/wal_encryption_ranges.per');" +); + +$node->psql('postgres', + "SELECT pg_tde_create_key_using_global_key_provider('server-key', 'file-keyring');" +); +$node->psql('postgres', + "SELECT pg_tde_set_server_key_using_global_key_provider('server-key', 'file-keyring');" +); + +$node->psql('postgres', 'SELECT pg_tde_verify_server_key();'); + +# Create test table and enable WAL encryption +$node->psql('postgres', + 'CREATE TABLE test_wal (id SERIAL, k INTEGER, PRIMARY KEY (id));'); + +$node->psql('postgres', 'ALTER SYSTEM SET pg_tde.wal_encrypt = on;'); + +$node->restart; + +# Insert some data to generate WAL records and check that WAL records are encrypted +$node->psql('postgres', 'INSERT INTO test_wal (k) VALUES (1), (2);'); + +my $enc_lsn = $node->safe_psql('postgres', "SELECT pg_current_wal_lsn();"); +$node->psql( + 'postgres', + "SELECT pg_tde_is_wal_record_encrypted('$enc_lsn'::pg_lsn);", + stdout => \$psql_out); +is($psql_out, 't', "Check that WAL record is encrypted"); + +# Force PG to switch to a new WAL segment to avoid situation when we decrypt +# non-full WAL page on WAL encryption off. +$node->psql('postgres', 'SELECT pg_switch_wal();'); +$node->psql('postgres', 'ALTER SYSTEM SET pg_tde.wal_encrypt = off;'); + +$node->restart; + +# Insert more data to generate WAL records and check that they are not encrypted +$node->psql('postgres', 'INSERT INTO test_wal (k) VALUES (3), (4);'); + +my $dec_lsn = $node->safe_psql('postgres', "SELECT pg_current_wal_lsn();"); +$node->psql( + 'postgres', + "SELECT pg_tde_is_wal_record_encrypted('$dec_lsn'::pg_lsn);", + stdout => \$psql_out); +is($psql_out, 'f', "Check that WAL record is not encrypted"); + +# Check that previously encrypted record is still encrypted +$node->psql( + 'postgres', + "SELECT pg_tde_is_wal_record_encrypted('$enc_lsn'::pg_lsn);", + stdout => \$psql_out); +is($psql_out, 't', "Check that WAL record is still encrypted"); + +# Check that WAL records that we recorded before match the encryption ranges. We use +# relative comparisons to avoid issues with LSN stability across different runs of the test. +my $ranges_count = $node->safe_psql( + 'postgres', "SELECT count(*) FROM pg_tde_get_wal_encryption_ranges() + WHERE start_lsn <= '$enc_lsn'::pg_lsn + AND end_lsn > '$enc_lsn'::pg_lsn + AND end_lsn <= '$dec_lsn'::pg_lsn + AND start_tli = 1 + AND end_tli = 1;"); +is($ranges_count, 1, + "Check that WAL records correspond to expected encryption range"); + +$node->psql('postgres', 'DROP EXTENSION pg_tde;'); +$node->stop; + +done_testing();