diff --git a/CHANGELOG.md b/CHANGELOG.md index 39a0a81ea..fe2d4b9d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Added - Support for PostgreSQL 19 Beta 3 ([PG-2536](https://perconadev.atlassian.net/browse/PG-2536)) +- Make pg_tde_waldump recognise and print proper data for the pg_tde custom resource manager ([PG-2754](https://perconadev.atlassian.net/browse/PG-2754)) ## 2.2.2 diff --git a/fetools/pg16/pg_waldump/pg_waldump.c b/fetools/pg16/pg_waldump/pg_waldump.c index 247a1cdd8..bf15ec092 100644 --- a/fetools/pg16/pg_waldump/pg_waldump.c +++ b/fetools/pg16/pg_waldump/pg_waldump.c @@ -33,6 +33,7 @@ #include "storage/bufpage.h" #include "pg_tde_fe.h" +#include "access/pg_tde_xlog.h" #include "access/pg_tde_xlog_smgr.h" #include "access/xlog_smgr.h" #include "catalog/tde_global_space.h" @@ -108,6 +109,8 @@ print_rmgr_list(void) { printf("%s\n", GetRmgrDesc(i)->rm_name); } + + printf("%s\n", RM_TDERMGR_NAME); } /* @@ -991,6 +994,11 @@ main(int argc, char **argv) config.filter_by_rmgr[rmid] = true; config.filter_by_rmgr_enabled = true; } + else if (pg_strcasecmp(optarg, RM_TDERMGR_NAME) == 0) + { + config.filter_by_rmgr[RM_TDERMGR_ID] = true; + config.filter_by_rmgr_enabled = true; + } else { /* then look for builtin rmgrs */ diff --git a/fetools/pg16/pg_waldump/rmgrdesc.c b/fetools/pg16/pg_waldump/rmgrdesc.c index 6b8c17bb4..97dc6b374 100644 --- a/fetools/pg16/pg_waldump/rmgrdesc.c +++ b/fetools/pg16/pg_waldump/rmgrdesc.c @@ -32,6 +32,8 @@ #include "storage/standbydefs.h" #include "utils/relmapper.h" +#include "access/pg_tde_xlog.h" + #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask,decode) \ { name, desc, identify}, @@ -80,6 +82,12 @@ initialize_custom_rmgrs(void) CustomRmgrDesc[i].rm_desc = default_desc; CustomRmgrDesc[i].rm_identify = default_identify; } + + /* Initialize pg_tde resource manager */ + CustomRmgrDesc[RM_TDERMGR_ID - RM_MIN_CUSTOM_ID].rm_name = RM_TDERMGR_NAME; + CustomRmgrDesc[RM_TDERMGR_ID - RM_MIN_CUSTOM_ID].rm_desc = tdeheap_rmgr_desc; + CustomRmgrDesc[RM_TDERMGR_ID - RM_MIN_CUSTOM_ID].rm_identify = tdeheap_rmgr_identify; + CustomRmgrDescInitialized = true; } diff --git a/fetools/pg17/pg_waldump/pg_waldump.c b/fetools/pg17/pg_waldump/pg_waldump.c index 0b6206408..4bfbee7dd 100644 --- a/fetools/pg17/pg_waldump/pg_waldump.c +++ b/fetools/pg17/pg_waldump/pg_waldump.c @@ -33,6 +33,7 @@ #include "storage/bufpage.h" #include "pg_tde_fe.h" +#include "access/pg_tde_xlog.h" #include "access/pg_tde_xlog_smgr.h" #include "access/xlog_smgr.h" #include "catalog/tde_global_space.h" @@ -108,6 +109,8 @@ print_rmgr_list(void) { printf("%s\n", GetRmgrDesc(i)->rm_name); } + + printf("%s\n", RM_TDERMGR_NAME); } /* @@ -995,6 +998,11 @@ main(int argc, char **argv) config.filter_by_rmgr[rmid] = true; config.filter_by_rmgr_enabled = true; } + else if (pg_strcasecmp(optarg, RM_TDERMGR_NAME) == 0) + { + config.filter_by_rmgr[RM_TDERMGR_ID] = true; + config.filter_by_rmgr_enabled = true; + } else { /* then look for builtin rmgrs */ diff --git a/fetools/pg17/pg_waldump/rmgrdesc.c b/fetools/pg17/pg_waldump/rmgrdesc.c index 6b8c17bb4..97dc6b374 100644 --- a/fetools/pg17/pg_waldump/rmgrdesc.c +++ b/fetools/pg17/pg_waldump/rmgrdesc.c @@ -32,6 +32,8 @@ #include "storage/standbydefs.h" #include "utils/relmapper.h" +#include "access/pg_tde_xlog.h" + #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask,decode) \ { name, desc, identify}, @@ -80,6 +82,12 @@ initialize_custom_rmgrs(void) CustomRmgrDesc[i].rm_desc = default_desc; CustomRmgrDesc[i].rm_identify = default_identify; } + + /* Initialize pg_tde resource manager */ + CustomRmgrDesc[RM_TDERMGR_ID - RM_MIN_CUSTOM_ID].rm_name = RM_TDERMGR_NAME; + CustomRmgrDesc[RM_TDERMGR_ID - RM_MIN_CUSTOM_ID].rm_desc = tdeheap_rmgr_desc; + CustomRmgrDesc[RM_TDERMGR_ID - RM_MIN_CUSTOM_ID].rm_identify = tdeheap_rmgr_identify; + CustomRmgrDescInitialized = true; } diff --git a/fetools/pg18/pg_waldump/pg_waldump.c b/fetools/pg18/pg_waldump/pg_waldump.c index 2b1712530..361ffce69 100644 --- a/fetools/pg18/pg_waldump/pg_waldump.c +++ b/fetools/pg18/pg_waldump/pg_waldump.c @@ -33,6 +33,7 @@ #include "storage/bufpage.h" #include "pg_tde_fe.h" +#include "access/pg_tde_xlog.h" #include "access/pg_tde_xlog_smgr.h" #include "access/xlog_smgr.h" #include "catalog/tde_global_space.h" @@ -108,6 +109,8 @@ print_rmgr_list(void) { printf("%s\n", GetRmgrDesc(i)->rm_name); } + + printf("%s\n", RM_TDERMGR_NAME); } /* @@ -995,6 +998,11 @@ main(int argc, char **argv) config.filter_by_rmgr[rmid] = true; config.filter_by_rmgr_enabled = true; } + else if (pg_strcasecmp(optarg, RM_TDERMGR_NAME) == 0) + { + config.filter_by_rmgr[RM_TDERMGR_ID] = true; + config.filter_by_rmgr_enabled = true; + } else { /* then look for builtin rmgrs */ diff --git a/fetools/pg18/pg_waldump/rmgrdesc.c b/fetools/pg18/pg_waldump/rmgrdesc.c index fac509ed1..2503d7d0b 100644 --- a/fetools/pg18/pg_waldump/rmgrdesc.c +++ b/fetools/pg18/pg_waldump/rmgrdesc.c @@ -32,6 +32,8 @@ #include "storage/standbydefs.h" #include "utils/relmapper.h" +#include "access/pg_tde_xlog.h" + #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask,decode) \ { name, desc, identify}, @@ -80,6 +82,12 @@ initialize_custom_rmgrs(void) CustomRmgrDesc[i].rm_desc = default_desc; CustomRmgrDesc[i].rm_identify = default_identify; } + + /* Initialize pg_tde resource manager */ + CustomRmgrDesc[RM_TDERMGR_ID - RM_MIN_CUSTOM_ID].rm_name = RM_TDERMGR_NAME; + CustomRmgrDesc[RM_TDERMGR_ID - RM_MIN_CUSTOM_ID].rm_desc = tdeheap_rmgr_desc; + CustomRmgrDesc[RM_TDERMGR_ID - RM_MIN_CUSTOM_ID].rm_identify = tdeheap_rmgr_identify; + CustomRmgrDescInitialized = true; } diff --git a/fetools/pg19/pg_waldump/pg_waldump.c b/fetools/pg19/pg_waldump/pg_waldump.c index d722ebf52..9d707d959 100644 --- a/fetools/pg19/pg_waldump/pg_waldump.c +++ b/fetools/pg19/pg_waldump/pg_waldump.c @@ -34,6 +34,7 @@ #include "storage/bufpage.h" #include "pg_tde_fe.h" +#include "access/pg_tde_xlog.h" #include "access/pg_tde_xlog_smgr.h" #include "access/xlog_smgr.h" #include "catalog/tde_global_space.h" @@ -102,6 +103,8 @@ print_rmgr_list(void) { printf("%s\n", GetRmgrDesc(i)->rm_name); } + + printf("%s\n", RM_TDERMGR_NAME); } /* @@ -1138,6 +1141,11 @@ main(int argc, char **argv) config.filter_by_rmgr[rmid] = true; config.filter_by_rmgr_enabled = true; } + else if (pg_strcasecmp(optarg, RM_TDERMGR_NAME) == 0) + { + config.filter_by_rmgr[RM_TDERMGR_ID] = true; + config.filter_by_rmgr_enabled = true; + } else { /* then look for builtin rmgrs */ diff --git a/fetools/pg19/pg_waldump/rmgrdesc.c b/fetools/pg19/pg_waldump/rmgrdesc.c index 931ab8b97..01e831c32 100644 --- a/fetools/pg19/pg_waldump/rmgrdesc.c +++ b/fetools/pg19/pg_waldump/rmgrdesc.c @@ -32,6 +32,8 @@ #include "storage/standbydefs.h" #include "utils/relmapper.h" +#include "access/pg_tde_xlog.h" + #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask,decode) \ { name, desc, identify}, @@ -80,6 +82,12 @@ initialize_custom_rmgrs(void) CustomRmgrDesc[i].rm_desc = default_desc; CustomRmgrDesc[i].rm_identify = default_identify; } + + /* Initialize pg_tde resource manager */ + CustomRmgrDesc[RM_TDERMGR_ID - RM_MIN_CUSTOM_ID].rm_name = RM_TDERMGR_NAME; + CustomRmgrDesc[RM_TDERMGR_ID - RM_MIN_CUSTOM_ID].rm_desc = tdeheap_rmgr_desc; + CustomRmgrDesc[RM_TDERMGR_ID - RM_MIN_CUSTOM_ID].rm_identify = tdeheap_rmgr_identify; + CustomRmgrDescInitialized = true; } diff --git a/meson.build b/meson.build index b42937e03..4aea0a6b6 100644 --- a/meson.build +++ b/meson.build @@ -153,6 +153,7 @@ pg_tde_frontend = static_library('pg_tde_frontend', files( 'src/pg_tde_fe.c', 'src/access/pg_tde_tdemap.c', + 'src/access/pg_tde_xlog.c', 'src/access/pg_tde_xlog_keys.c', 'src/catalog/tde_keyring.c', 'src/catalog/tde_keyring_parse_opts.c', @@ -378,6 +379,7 @@ tap_tests = [ 't/pg_tde_upgrade.pl', 't/pg_waldump_basic.pl', 't/pg_waldump_fullpage.pl', + 't/pg_waldump_tde_rmgr.pl', 't/regress.pl', 't/replication.pl', 't/reuse_relfilenode_in_cache.pl', diff --git a/src/access/pg_tde_xlog.c b/src/access/pg_tde_xlog.c index abace3014..916141e95 100644 --- a/src/access/pg_tde_xlog.c +++ b/src/access/pg_tde_xlog.c @@ -4,6 +4,7 @@ #include "postgres.h" +#ifndef FRONTEND #include "access/xlog.h" #include "access/xlog_internal.h" #include "access/xloginsert.h" @@ -11,6 +12,7 @@ #include "storage/shmem.h" #include "utils/guc.h" #include "utils/memutils.h" +#endif #include "access/pg_tde_xlog_keys.h" #include "access/pg_tde_xlog.h" @@ -21,12 +23,14 @@ #include "pg_tde_defines.h" #include "smgr/pg_tde_smgr.h" +#ifdef FRONTEND +#include "pg_tde_fe.h" +#else + static void tdeheap_rmgr_redo(XLogReaderState *record); -static void tdeheap_rmgr_desc(StringInfo buf, XLogReaderState *record); -static const char *tdeheap_rmgr_identify(uint8 info); static const RmgrData tdeheap_rmgr = { - .rm_name = "pg_tde", + .rm_name = RM_TDERMGR_NAME, .rm_redo = tdeheap_rmgr_redo, .rm_desc = tdeheap_rmgr_desc, .rm_identify = tdeheap_rmgr_identify, @@ -93,8 +97,9 @@ tdeheap_rmgr_redo(XLogReaderState *record) elog(PANIC, "pg_tde_redo: unknown op code %u", info); } } +#endif /* !FRONTEND */ -static void +void tdeheap_rmgr_desc(StringInfo buf, XLogReaderState *record) { uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; @@ -143,7 +148,7 @@ tdeheap_rmgr_desc(StringInfo buf, XLogReaderState *record) } } -static const char * +const char * tdeheap_rmgr_identify(uint8 info) { switch (info & ~XLR_INFO_MASK) diff --git a/src/include/access/pg_tde_xlog.h b/src/include/access/pg_tde_xlog.h index eba8319ab..771e1de28 100644 --- a/src/include/access/pg_tde_xlog.h +++ b/src/include/access/pg_tde_xlog.h @@ -5,7 +5,7 @@ #ifndef PG_TDE_XLOG_H #define PG_TDE_XLOG_H -#ifndef FRONTEND +#include "access/xlogreader.h" /* TDE XLOG record types */ #define XLOG_TDE_CREATE_RELATION_KEY 0x00 @@ -18,13 +18,18 @@ /* ID 140 is registered for Percona TDE extension: https://wiki.postgresql.org/wiki/CustomWALResourceManagers */ #define RM_TDERMGR_ID 140 +#define RM_TDERMGR_NAME "pg_tde" typedef struct XLogRelKey { RelFileLocator rlocator; } XLogRelKey; -extern void RegisterTdeRmgr(void); +extern void tdeheap_rmgr_desc(StringInfo buf, XLogReaderState *record); +extern const char *tdeheap_rmgr_identify(uint8 info); +#ifndef FRONTEND +extern void RegisterTdeRmgr(void); #endif /* !FRONTEND */ + #endif /* PG_TDE_XLOG_H */ diff --git a/t/pg_waldump_tde_rmgr.pl b/t/pg_waldump_tde_rmgr.pl new file mode 100644 index 000000000..ae0d1c7ac --- /dev/null +++ b/t/pg_waldump_tde_rmgr.pl @@ -0,0 +1,110 @@ +# Tests that pg_tde_waldump recognises the pg_tde custom resource manager: +# it must print "rmgr: pg_tde" with a proper description instead of the +# generic "rmgr: custom140 ... desc: UNKNOWN (%d) rmid: %d", and it must +# accept "--rmgr=pg_tde" (as well as the legacy "--rmgr=custom140") to +# filter WAL records. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $keydir = PostgreSQL::Test::Utils::tempdir; + +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf( + 'postgresql.conf', q{ +autovacuum = off +checkpoint_timeout = 1h +shared_preload_libraries = 'pg_tde' +}); +$node->start; + +my $start_lsn = + $node->safe_psql('postgres', q{SELECT pg_current_wal_insert_lsn()}); + +# Generate pg_tde rmgr WAL records: XLOG_TDE_INSTALL_EXTENSION, +# XLOG_TDE_WRITE_KEY_PROVIDER, XLOG_TDE_ADD_PRINCIPAL_KEY. +$node->safe_psql('postgres', "CREATE EXTENSION pg_tde;"); +$node->safe_psql('postgres', + "SELECT pg_tde_add_global_key_provider_file('file-keyring-wal', '$keydir/global.keys');" +); +$node->safe_psql('postgres', + "SELECT pg_tde_create_key_using_global_key_provider('server-key', 'file-keyring-wal');" +); +$node->safe_psql('postgres', + "SELECT pg_tde_set_server_key_using_global_key_provider('server-key', 'file-keyring-wal');" +); + +my $end_lsn = + $node->safe_psql('postgres', q{SELECT pg_current_wal_insert_lsn()}); + +$node->stop; + +# Helper function to test various options. Pass options as arguments. +# Output lines are returned as array. Copy of the func from pg_waldump_basic.pl +sub test_pg_waldump +{ + local $Test::Builder::Level = $Test::Builder::Level + 1; + my @opts = @_; + + my (@cmd, $stdout, $stderr, $result, @lines); + + @cmd = ( + 'pg_tde_waldump', '-k', + $node->data_dir . '/pg_tde', '-p', + $node->data_dir, '--start', + $start_lsn, '--end', + $end_lsn); + push @cmd, @opts; + $result = IPC::Run::run \@cmd, '>', \$stdout, '2>', \$stderr; + ok($result, "pg_tde_waldump @opts: runs ok"); + is($stderr, '', "pg_tde_waldump @opts: no stderr"); + @lines = split /\n/, $stdout; + ok(@lines > 0, "pg_tde_waldump @opts: some lines are output"); + return @lines; +} + +my @lines; + +# rmgr name and description are resolved, not left as the generic +# "custom140" / "UNKNOWN" output. +@lines = test_pg_waldump; +ok((grep { /^rmgr: pg_tde\b/ } @lines), 'pg_tde rmgr lines are present'); +is(scalar(grep { /^rmgr: custom140\b/ } @lines), + 0, 'no unresolved custom140 rmgr lines'); +is(scalar(grep { /^rmgr: pg_tde\b.*\bdesc: UNKNOWN\b/ } @lines), + 0, 'no unresolved UNKNOWN description for pg_tde records'); +ok((grep { /^rmgr: pg_tde\b.*\bdesc: INSTALL_EXTENSION\b/ } @lines), + 'INSTALL_EXTENSION record is described'); +ok((grep { /^rmgr: pg_tde\b.*\bdesc: WRITE_KEY_PROVIDER\b/ } @lines), + 'WRITE_KEY_PROVIDER record is described'); +ok((grep { /^rmgr: pg_tde\b.*\bdesc: ADD_PRINCIPAL_KEY\b/ } @lines), + 'ADD_PRINCIPAL_KEY record is described'); + +# --rmgr=list includes the pg_tde resource manager by name. +{ + my ($stdout, $stderr); + my $result = IPC::Run::run [ 'pg_tde_waldump', '--rmgr', 'list' ], '>', + \$stdout, '2>', \$stderr; + ok($result, 'pg_tde_waldump --rmgr=list runs ok'); + like($stdout, qr/^pg_tde$/m, 'pg_tde is listed as a known rmgr'); +} + +# Filtering by the friendly name "pg_tde" only returns pg_tde lines, and +# matches the set of lines returned when filtering by the legacy +# "custom140" name. +@lines = test_pg_waldump('--rmgr', 'pg_tde'); +ok(@lines > 0, 'at least one pg_tde record found when filtering by name'); +is(scalar(grep { !/^rmgr: pg_tde\b/ } @lines), + 0, 'only pg_tde lines with --rmgr=pg_tde'); + +my @lines_legacy = test_pg_waldump('--rmgr', 'custom140'); +is(scalar(grep { !/^rmgr: pg_tde\b/ } @lines_legacy), + 0, 'only pg_tde lines with legacy --rmgr=custom140'); +is_deeply(\@lines, \@lines_legacy, + '--rmgr=pg_tde and --rmgr=custom140 select the same records'); + +done_testing();