PG-1870 Run postgresql TAP suite with pg_tde enabled

ci_scripts/tde_setup.sql

@@ -1,6 +1,6 @@
-CREATE SCHEMA IF NOT EXISTS tde;
-CREATE EXTENSION IF NOT EXISTS pg_tde SCHEMA tde;
+CREATE SCHEMA IF NOT EXISTS _pg_tde;
+CREATE EXTENSION IF NOT EXISTS pg_tde SCHEMA _pg_tde;
 \! rm -f '/tmp/pg_tde_test_keyring.per'
-SELECT tde.pg_tde_add_database_key_provider_file('reg_file-vault', '/tmp/pg_tde_test_keyring.per');
-SELECT tde.pg_tde_create_key_using_database_key_provider('test-db-key', 'reg_file-vault');
-SELECT tde.pg_tde_set_key_using_database_key_provider('test-db-key', 'reg_file-vault');
+SELECT _pg_tde.pg_tde_add_database_key_provider_file('reg_file-vault', '/tmp/pg_tde_test_keyring.per');
+SELECT _pg_tde.pg_tde_create_key_using_database_key_provider('test-db-key', 'reg_file-vault');
+SELECT _pg_tde.pg_tde_set_key_using_database_key_provider('test-db-key', 'reg_file-vault');

ci_scripts/tde_setup_global.sql

@@ -1,11 +1,10 @@
-CREATE SCHEMA tde;
-CREATE EXTENSION IF NOT EXISTS pg_tde SCHEMA tde;
+CREATE SCHEMA IF NOT EXISTS _pg_tde;
+CREATE EXTENSION IF NOT EXISTS pg_tde SCHEMA _pg_tde;
 
 \! rm -f '/tmp/pg_tde_test_keyring.per'
-SELECT tde.pg_tde_add_global_key_provider_file('reg_file-global', '/tmp/pg_tde_test_keyring.per');
-SELECT tde.pg_tde_create_key_using_global_key_provider('server-key', 'reg_file-global');
-SELECT tde.pg_tde_set_server_key_using_global_key_provider('server-key', 'reg_file-global');
+SELECT _pg_tde.pg_tde_add_global_key_provider_file('reg_file-global', '/tmp/pg_tde_test_keyring.per');
+SELECT _pg_tde.pg_tde_create_key_using_global_key_provider('server-key', 'reg_file-global');
+SELECT _pg_tde.pg_tde_set_server_key_using_global_key_provider('server-key', 'reg_file-global');
 ALTER SYSTEM SET pg_tde.wal_encrypt = on;
 ALTER SYSTEM SET default_table_access_method = 'tde_heap';
-ALTER SYSTEM SET search_path = "$user",public,tde;
 -- restart required

contrib/amcheck/t/001_verify_heapam.pl

@@ -9,6 +9,11 @@
 
 use Test::More;
 
+if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all => "hacks relation files directly for scaffolding";
+}
+
 my ($node, $result);
 
 #

meson.build

@@ -3445,6 +3445,48 @@ sys.exit(sp.returncode)
      env: test_env,
      suite: ['setup'])
 
+test_tde_template_dir = test_install_destdir / 'pg_tde_template'
+test_env.set('TDE_TEMPLATE_DIR', test_tde_template_dir)
+
+test('init_tde_files',
+     find_program('sh', required: true, native: true),
+     args: [
+       '-c', '''
+set -e
+
+if [ -z "$TDE_MODE" ]; then
+	exit;
+fi
+
+set -e
+
+PATH="$1":$PATH
+TMP_DATA_DIR=$(mktemp -d)
+
+rm -rf "$2"
+mkdir "$2"
+
+pg_ctl -D "$TMP_DATA_DIR" init -o '--set shared_preload_libraries=pg_tde'
+
+postgres --single -F -j -D "$TMP_DATA_DIR" postgres << SQL
+	CREATE EXTENSION pg_tde;
+	SELECT pg_tde_add_global_key_provider_file('global_test_provider', '$2/pg_tde_test_keys');
+	SELECT pg_tde_create_key_using_global_key_provider('test_default_key', 'global_test_provider');
+	SELECT pg_tde_set_default_key_using_global_key_provider('test_default_key', 'global_test_provider');
+SQL
+
+cp -RPp "$TMP_DATA_DIR/pg_tde" "$2/pg_tde"
+rm -rf "$TMP_DATA_DIR"
+''',
+       'init_tde_files',
+       temp_install_bindir,
+       test_tde_template_dir
+     ],
+     priority: setup_tests_priority - 2,
+     timeout: 300,
+     is_parallel: false,
+     env: test_env,
+     suite: ['setup'])
 
 
 ###############################################################

src/bin/pg_amcheck/t/003_check.pl

@@ -9,6 +9,11 @@
 
 use Test::More;
 
+if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all => "hacks relation files directly for scaffolding";
+}
+
 my ($node, $port, %corrupt_page, %remove_relation);
 
 # Returns the filesystem path for the named relation.

src/bin/pg_amcheck/t/005_opclass_damage.pl

@@ -10,6 +10,11 @@
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all => 'investigate why this fails';
+}
+
 my $node = PostgreSQL::Test::Cluster->new('test');
 $node->init;
 $node->start;

src/bin/pg_basebackup/t/010_pg_basebackup.pl

@@ -10,6 +10,18 @@
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{TDE_MODE_WAL} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all =>
+	  "pg_basebackup without -E from server with encrypted WAL produces broken backups";
+}
+
+if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all =>
+	  'uses corrupt_page_checksum to directly hack relation files';
+}
+
 program_help_ok('pg_basebackup');
 program_version_ok('pg_basebackup');
 program_options_handling_ok('pg_basebackup');

src/bin/pg_checksums/t/002_actions.pl

@@ -12,6 +12,11 @@
 
 use Test::More;
 
+if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all =>
+	  'uses corrupt_page_checksum to directly hack relation files';
+}
 
 # Utility routine to create and check a table with corrupted checksums
 # on a wanted tablespace.  Note that this stops and starts the node

src/bin/pg_combinebackup/t/003_timeline.pl

@@ -10,6 +10,12 @@
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{TDE_MODE_WAL} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all =>
+	  "pg_basebackup without -E from server with encrypted WAL produces broken backups";
+}
+
 # Can be changed to test the other modes.
 my $mode = $ENV{PG_TEST_PG_COMBINEBACKUP_MODE} || '--copy';
 

src/bin/pg_combinebackup/t/006_db_file_copy.pl

@@ -7,6 +7,12 @@
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{TDE_MODE_WAL} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all =>
+	  "pg_basebackup without -E from server with encrypted WAL produces broken backups";
+}
+
 # Can be changed to test the other modes.
 my $mode = $ENV{PG_TEST_PG_COMBINEBACKUP_MODE} || '--copy';
 

src/bin/pg_combinebackup/t/008_promote.pl

@@ -10,6 +10,12 @@
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{TDE_MODE_WAL} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all =>
+	  "pg_basebackup without -E from server with encrypted WAL produces broken backups";
+}
+
 # Can be changed to test the other modes.
 my $mode = $ENV{PG_TEST_PG_COMBINEBACKUP_MODE} || '--copy';
 

src/bin/pg_dump/t/004_pg_dump_parallel.pl

@@ -8,6 +8,12 @@
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all =>
+	  'pg_restore fail to restore _pg_tde schema on cluster which already has it';
+}
+
 my $dbname1 = 'regression_src';
 my $dbname2 = 'regression_dest1';
 my $dbname3 = 'regression_dest2';

src/bin/pg_dump/t/010_dump_connstr.pl

@@ -8,6 +8,12 @@
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all =>
+	  'pg_restore fail to restore _pg_tde schema on cluster which already has it';
+}
+
 if ($PostgreSQL::Test::Utils::is_msys2)
 {
 	plan skip_all => 'High bit name tests fail on Msys2';

src/bin/pg_rewind/t/001_basic.pl

@@ -11,6 +11,12 @@
 
 use RewindTest;
 
+if ($ENV{TDE_MODE_WAL} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all =>
+	  "copies WAL directly to archive without using archive_command";
+}
+
 sub run_test
 {
 	my $test_mode = shift;

src/bin/pg_rewind/t/002_databases.pl

@@ -11,6 +11,12 @@
 
 use RewindTest;
 
+if ($ENV{TDE_MODE} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all =>
+	  "pg_combinebackup doesn't set filemodes of pg_tde/ correctly?";
+}
+
 sub run_test
 {
 	my $test_mode = shift;

src/bin/pg_upgrade/t/002_pg_upgrade.pl

@@ -15,9 +15,10 @@
 use PostgreSQL::Test::AdjustUpgrade;
 use Test::More;
 
-if (defined($ENV{TDE_MODE}))
+if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP})
 {
-    plan skip_all => "Running with TDE doesn't support special server starts yet";
+	plan skip_all =>
+	  'pg_restore fail to restore _pg_tde schema on cluster which already has it';
 }
 
 # Can be changed to test the other modes.

src/bin/pg_upgrade/t/003_logical_slots.pl

@@ -11,6 +11,12 @@
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all =>
+	  'pg_restore fail to restore _pg_tde schema on cluster which already has it';
+}
+
 # Can be changed to test the other modes
 my $mode = $ENV{PG_TEST_PG_UPGRADE_MODE} || '--copy';
 

src/bin/pg_upgrade/t/004_subscription.pl

@@ -12,6 +12,12 @@
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all =>
+	  'pg_restore fail to restore _pg_tde schema on cluster which already has it';
+}
+
 # Can be changed to test the other modes.
 my $mode = $ENV{PG_TEST_PG_UPGRADE_MODE} || '--copy';
 

src/bin/pg_verifybackup/t/009_extract.pl

@@ -10,6 +10,13 @@
 use PostgreSQL::Test::Cluster;
 use PostgreSQL::Test::Utils;
 use Test::More;
+
+if ($ENV{TDE_MODE_WAL} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all =>
+	  "pg_basebackup without -E from server with encrypted WAL produces broken backups";
+}
+
 my $primary = PostgreSQL::Test::Cluster->new('primary');
 $primary->init(allows_streaming => 1);
 $primary->start;

src/bin/pg_waldump/t/001_basic.pl

@@ -7,6 +7,11 @@
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{TDE_MODE_WAL} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all => "pg_waldump needs extra options for encrypted WAL";
+}
+
 program_help_ok('pg_waldump');
 program_version_ok('pg_waldump');
 program_options_handling_ok('pg_waldump');

src/bin/pg_waldump/t/002_save_fullpage.pl

@@ -9,6 +9,11 @@
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{TDE_MODE_WAL} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all => "pg_waldump needs extra options for encrypted WAL";
+}
+
 my ($blocksize, $walfile_name);
 
 # Function to extract the LSN from the given block structure

src/bin/scripts/t/020_createdb.pl

@@ -8,6 +8,12 @@
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{TDE_MODE_SMGR} and not $ENV{TDE_MODE_NOSKIP})
+{
+	plan skip_all =>
+	  'tries to use FILE_COPY strategy for database creation with encrypted objects in the template';
+}
+
 program_help_ok('createdb');
 program_version_ok('createdb');
 program_options_handling_ok('createdb');

src/test/perl/PostgreSQL/Test/Cluster.pm

@@ -107,6 +107,7 @@ use File::Temp ();
 use IPC::Run;
 use PostgreSQL::Version;
 use PostgreSQL::Test::RecursiveCopy;
+use PostgreSQL::Test::TdeCluster;
 use Socket;
 use Test::More;
 use PostgreSQL::Test::Utils          ();
@@ -1527,6 +1528,11 @@ sub new
 		}
 	}
 
+	if ($ENV{TDE_MODE})
+	{
+		bless $node, 'PostgreSQL::Test::TdeCluster';
+	}
+
 	# Add node to list of nodes
 	push(@all_nodes, $node);