diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 3538e0cdc757c..7e1c817416c9c 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -14,6 +14,9 @@ # # $ git log --pretty=format:"%H # %cd%n# %s" $PGINDENTGITHASH -1 --date=iso +71da1f03f2f0ff18ed11e4ba6b07b6bd56705a5d # 2025-04-22 11:40:24 +0200 +# Run pgperltidy + c739ae9e288c095cfe1b91ce27a2f2c075ed5da4 # 2024-08-26 16:16:09 -0700 # Fix identation. diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index faa87493bf62c..3bdd7c5f42812 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,5 +2,5 @@ # Order is important; the last matching pattern takes the most precedence. * @dutow @dAdAbird -/contrib/pg_tde/documentation/ @nastena1606 +/contrib/pg_tde/documentation/ @nastena1606 @Andriciuc /.github/ @artemgavrilov diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 0000000000000..e124140442877 --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,45 @@ +codecov: + strict_yaml_branch: TDE_REL_17_STABLE +fixes: + - "src/::contrib/pg_tde/src/" # move path for codecov file mappings e.g., "src/" => "contrib/pg_tde/src/" +coverage: + status: + project: + default: + target: 90% + threshold: 1% + base: auto +comment: + layout: "header, diff, components" +component_management: + default_rules: + statuses: + - type: project + target: auto + branches: + - "TDE_REL_17_STABLE" + individual_components: + - component_id: access + paths: + - contrib/pg_tde/src/access/** + - component_id: catalog + paths: + - contrib/pg_tde/src/catalog/** + - component_id: common + paths: + - contrib/pg_tde/src/common/** + - component_id: encryption + paths: + - contrib/pg_tde/src/encryption/** + - component_id: keyring + paths: + - contrib/pg_tde/src/keyring/** + - component_id: src + paths: + - contrib/pg_tde/src/*.c + - component_id: smgr + paths: + - contrib/pg_tde/src/smgr/** + - component_id: transam + paths: + - contrib/pg_tde/src/transam/** diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000000000..b01c01826db24 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,56 @@ +name: Code coverage +on: + pull_request: + push: + branches: + - TDE_REL_17_STABLE + +jobs: + collect: + name: Collect and upload + runs-on: ubuntu-22.04 + steps: + - name: Clone repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install dependencies + run: ci_scripts/ubuntu-deps.sh + + - name: Build postgres + run: ci_scripts/make-build.sh debug --enable-coverage + + - name: Setup kmip and vault + run: ci_scripts/setup-keyring-servers.sh + + - name: Test postgres with TDE to generate coverage + run: ci_scripts/make-test.sh --tde-only + + - name: Collect coverage data + run: find . -type f -name "*.c" ! -path '*libkmip*' | xargs -t gcov -abcfu + working-directory: contrib/pg_tde + + - name: Upload coverage data to codecov.io + uses: codecov/codecov-action@v5 + with: + verbose: true + token: ${{ secrets.CODECOV_TOKEN }} + working-directory: contrib/pg_tde + files: "*.c.gcov" + + - name: Report on test fail + uses: actions/upload-artifact@v4 + if: ${{ failure() }} + with: + name: coverage-testlog-tde + path: | + build/testrun/ + contrib/pg_tde/t/ + contrib/pg_tde/results + contrib/pg_tde/regression.diffs + contrib/pg_tde/regression.out + contrib/pg_tde/*.gcov + retention-days: 3 + + diff --git a/.github/workflows/pgindent.yml b/.github/workflows/pgindent.yml index 211d255f07042..c65ee645a8ba2 100644 --- a/.github/workflows/pgindent.yml +++ b/.github/workflows/pgindent.yml @@ -10,7 +10,7 @@ defaults: jobs: check: name: Check - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Clone repository uses: actions/checkout@v4 @@ -26,6 +26,9 @@ jobs: - name: Update typedefs run: ci_scripts/dump-typedefs.sh - + - name: Run pgindent run: ci_scripts/run-pgindent.sh --check --diff + + - name: Run pgperltidy + run: ci_scripts/run-pgperltidy.sh --assert-tidy --standard-error-output diff --git a/.github/workflows/psp-matrix.yml b/.github/workflows/psp-matrix.yml index 64a6c6babd5d1..ec64492a20913 100644 --- a/.github/workflows/psp-matrix.yml +++ b/.github/workflows/psp-matrix.yml @@ -1,6 +1,9 @@ name: PSP on: pull_request: + push: + branches: + - TDE_REL_17_STABLE workflow_dispatch: jobs: diff --git a/.github/workflows/psp-reusable.yml b/.github/workflows/psp-reusable.yml index 48293fbe21324..44742bab3ffc1 100644 --- a/.github/workflows/psp-reusable.yml +++ b/.github/workflows/psp-reusable.yml @@ -33,7 +33,7 @@ jobs: - name: Build postgres run: src/ci_scripts/${{ inputs.build_script }}-build.sh ${{ inputs.build_type }} - - name: 'Tar files' + - name: Archive pginst to artifact tar file run: tar -czf artifacts.tar src pginst - name: Upload build artifacts @@ -57,7 +57,7 @@ jobs: name: ${{ env.artifact_name }} path: . - - name: 'Untar files' + - name: Extract artifact file run: tar -xzf artifacts.tar - name: Install dependencies @@ -73,13 +73,20 @@ jobs: uses: actions/upload-artifact@v4 if: ${{ failure() }} with: - name: testlog-ubuntu-${{ inputs.ubuntu_version }}.04-meson-${{ inputs.build_type }} + name: log-test-${{ inputs.os }}-${{ inputs.build_script }}-${{ inputs.build_type }} path: | src/build/testrun/ - src/contrib/*/t/ - src/contrib/*/results + src/contrib/*/log src/contrib/*/regression.diffs src/contrib/*/regression.out + src/contrib/*/results + src/contrib/*/tmp_check + src/contrib/*/t/results + src/src/test/*/log + src/src/test/*/regression.diffs + src/src/test/*/regression.out + src/src/test/*/results + src/src/test/*/tmp_check retention-days: 3 test_tde: @@ -95,27 +102,34 @@ jobs: name: ${{ env.artifact_name }} path: . - - name: 'Untar files' + - name: Extract artifact file run: tar -xzf artifacts.tar - name: Install dependencies - run: src/ci_scripts/ubuntu-deps.sh - + run: src/ci_scripts/ubuntu-deps.sh + - name: Setup kmip and vault run: src/ci_scripts/setup-keyring-servers.sh - - - name: Test postgres with TDE - run: src/ci_scripts/${{ inputs.build_script }}-test-tde.sh --continue + + - name: Test postgres with TDE as default access method + run: src/ci_scripts/${{ inputs.build_script }}-test-global-tde.sh --continue - name: Report on test fail uses: actions/upload-artifact@v4 if: ${{ failure() }} with: - name: testlog-tde-${{ inputs.os }}-${{ inputs.build_script }}-${{ inputs.build_type }} + name: log-test-global-tde-${{ inputs.os }}-${{ inputs.build_script }}-${{ inputs.build_type }} path: | src/build/testrun/ - src/contrib/*/t/ - src/contrib/*/results + src/contrib/*/log src/contrib/*/regression.diffs src/contrib/*/regression.out + src/contrib/*/results + src/contrib/*/tmp_check + src/contrib/*/t/results + src/src/test/*/log + src/src/test/*/regression.diffs + src/src/test/*/regression.out + src/src/test/*/results + src/src/test/*/tmp_check retention-days: 3 diff --git a/README.md b/README.md index 835ece8089a18..4501f53904c77 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +### Code coverage for [pg_tde](https://github.com/percona/postgres/tree/TDE_REL_17_STABLE/contrib/pg_tde): [![codecov](https://codecov.io/github/percona/postgres/graph/badge.svg?token=Wow78BMYdP)](https://codecov.io/github/percona/postgres) + Percona Server for PostgreSQL ============================= diff --git a/ci_scripts/backup/pg_basebackup_test.sh b/ci_scripts/backup/pg_basebackup_test.sh index d9424be77c24a..a9b4f4199fe95 100755 --- a/ci_scripts/backup/pg_basebackup_test.sh +++ b/ci_scripts/backup/pg_basebackup_test.sh @@ -104,8 +104,8 @@ setup_tde_heap(){ sudo -u "$PG_USER" psql -p $PG_PORT -c "DROP DATABASE IF EXISTS $DB_NAME;" sudo -u "$PG_USER" psql -p $PG_PORT -c "CREATE DATABASE $DB_NAME;" sudo -u "$PG_USER" psql -d "$DB_NAME" -p "$PG_PORT" -c "CREATE EXTENSION IF NOT EXISTS pg_tde;" - sudo -u "$PG_USER" psql -d "$DB_NAME" -p "$PG_PORT" -c "SELECT pg_tde_add_key_provider_file('file-vault','$KEYLOCATION');" - sudo -u "$PG_USER" psql -d "$DB_NAME" -p "$PG_PORT" -c "SELECT pg_tde_set_principal_key('test-db-master-key','file-vault');" + sudo -u "$PG_USER" psql -d "$DB_NAME" -p "$PG_PORT" -c "SELECT pg_tde_add_database_key_provider_file('file-vault','$KEYLOCATION');" + sudo -u "$PG_USER" psql -d "$DB_NAME" -p "$PG_PORT" -c "SELECT pg_tde_set_key_using_database_key_provider('test-db-master-key','file-vault');" sudo -u "$PG_USER" psql -p $PG_PORT -c "ALTER DATABASE $DB_NAME SET default_table_access_method='tde_heap';" sudo -u "$PG_USER" psql -p $PG_PORT -c "SELECT pg_reload_conf();" } diff --git a/ci_scripts/configure-tde-server.sh b/ci_scripts/configure-global-tde.sh similarity index 90% rename from ci_scripts/configure-tde-server.sh rename to ci_scripts/configure-global-tde.sh index a5dfedd4c42bc..40d4cdbd26dd5 100644 --- a/ci_scripts/configure-tde-server.sh +++ b/ci_scripts/configure-global-tde.sh @@ -1,14 +1,14 @@ #!/bin/bash set -e -# This script is used to configure a TDE server for testing purposes. -export TDE_MODE=1 - SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)" INSTALL_DIR="$SCRIPT_DIR/../../pginst" +source $SCRIPT_DIR/env.sh cd "$SCRIPT_DIR/.." +source "$SCRIPT_DIR/env.sh" +export TDE_MODE=1 export PATH=$INSTALL_DIR/bin:$PATH export DATA_DIR=$INSTALL_DIR/data export PGDATA="${1:-$DATA_DIR}" diff --git a/ci_scripts/dump-typedefs.sh b/ci_scripts/dump-typedefs.sh index 181beac31c07c..6f854a4889570 100755 --- a/ci_scripts/dump-typedefs.sh +++ b/ci_scripts/dump-typedefs.sh @@ -1,33 +1,14 @@ #!/bin/bash SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)" -cd "$SCRIPT_DIR/../" - -if ! test -f src/backend/postgres; then - echo "src/backend/postgres doesn't exists, run make-build.sh first in debug mode" - exit 1 -fi +cd "$SCRIPT_DIR/.." if ! test -f contrib/pg_tde/pg_tde.so; then echo "contrib/pg_tde/pg_tde.so doesn't exists, run make-build.sh first in debug mode" exit 1 fi -objdump -W src/backend/postgres |\ - egrep -A3 DW_TAG_typedef |\ - perl -e ' while (<>) { chomp; @flds = split;next unless (1 < @flds);\ - next if $flds[0] ne "DW_AT_name" && $flds[1] ne "DW_AT_name";\ - next if $flds[-1] =~ /^DW_FORM_str/;\ - print $flds[-1],"\n"; }' |\ - sort | uniq > percona.typedefs - -objdump -W contrib/pg_tde/pg_tde.so |\ - egrep -A3 DW_TAG_typedef |\ - perl -e ' while (<>) { chomp; @flds = split;next unless (1 < @flds);\ - next if $flds[0] ne "DW_AT_name" && $flds[1] ne "DW_AT_name";\ - next if $flds[-1] =~ /^DW_FORM_str/;\ - print $flds[-1],"\n"; }' |\ - sort | uniq > tde.typedefs +src/tools/find_typedef contrib/pg_tde > pg_tde.typedefs # Combine with original typedefs -cat percona.typedefs tde.typedefs src/tools/pgindent/typedefs.list | sort | uniq > combined.typedefs +cat pg_tde.typedefs src/tools/pgindent/typedefs.list | sort -u > combined.typedefs diff --git a/ci_scripts/env.sh b/ci_scripts/env.sh new file mode 100644 index 0000000000000..1c729ccfd21e4 --- /dev/null +++ b/ci_scripts/env.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +export PERCONA_SERVER_VERSION=17.4.1 diff --git a/ci_scripts/make-build.sh b/ci_scripts/make-build.sh index 48e015f85af8b..c99e77a051bb1 100755 --- a/ci_scripts/make-build.sh +++ b/ci_scripts/make-build.sh @@ -1,9 +1,19 @@ #!/bin/bash -export TDE_MODE=1 +ENABLE_COVERAGE= + +for arg in "$@" +do + case "$arg" in + --enable-coverage) + ENABLE_COVERAGE="--enable-coverage" + shift;; + esac +done SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)" INSTALL_DIR="$SCRIPT_DIR/../../pginst" +source "$SCRIPT_DIR/env.sh" cd "$SCRIPT_DIR/.." @@ -12,5 +22,5 @@ if [ "$1" = "debugoptimized" ]; then export CXXFLAGS="-O2" fi -./configure --enable-debug --enable-cassert --enable-tap-tests --prefix=$INSTALL_DIR +./configure --prefix="$INSTALL_DIR" --enable-debug --enable-cassert --enable-tap-tests $ENABLE_COVERAGE make install-world -j diff --git a/ci_scripts/make-test-tde.sh b/ci_scripts/make-test-global-tde.sh similarity index 54% rename from ci_scripts/make-test-tde.sh rename to ci_scripts/make-test-global-tde.sh index 161feff01b99a..ee4a8787fbf5c 100755 --- a/ci_scripts/make-test-tde.sh +++ b/ci_scripts/make-test-global-tde.sh @@ -1,16 +1,18 @@ #!/bin/bash set -e +ADD_FLAGS= -export TDE_MODE=1 +for arg in "$@" +do + case "$arg" in + --continue) + ADD_FLAGS="-k" + shift;; + esac +done SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)" -source $SCRIPT_DIR/configure-tde-server.sh - -ADD_FLAGS= - -if [ "$1" = "--continue" ]; then - ADD_FLAGS="-k" -fi +source "$SCRIPT_DIR/configure-global-tde.sh" EXTRA_REGRESS_OPTS="--extra-setup=$SCRIPT_DIR/tde_setup.sql" make -s installcheck-world $ADD_FLAGS diff --git a/ci_scripts/make-test.sh b/ci_scripts/make-test.sh index e16ef4a986948..6ce136dd28bca 100755 --- a/ci_scripts/make-test.sh +++ b/ci_scripts/make-test.sh @@ -1,10 +1,25 @@ #!/bin/bash -export TDE_MODE=1 +set -e +TDE_ONLY=0 -SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P)" -INSTALL_DIR="$SCRIPT_DIR/../../pginst" +for arg in "$@" +do + case "$arg" in + --tde-only) + TDE_ONLY=1 + shift;; + esac +done -cd "$SCRIPT_DIR/.." +SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)" +source "$SCRIPT_DIR/env.sh" -make -s check-world +if [ "$TDE_ONLY" -eq 1 ]; +then + cd "$SCRIPT_DIR/../contrib/pg_tde" + make -s check +else + cd "$SCRIPT_DIR/.." + make -s check-world +fi diff --git a/ci_scripts/meson-build.sh b/ci_scripts/meson-build.sh index c94b12421ab61..e954b04619c09 100755 --- a/ci_scripts/meson-build.sh +++ b/ci_scripts/meson-build.sh @@ -1,8 +1,21 @@ #!/bin/bash +ENABLE_COVERAGE= + +for arg in "$@" +do + case "$arg" in + --enable-coverage) + ENABLE_COVERAGE="-Db_coverage=true" + shift;; + esac +done + SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)" +INSTALL_DIR="$SCRIPT_DIR/../../pginst" +source "$SCRIPT_DIR/env.sh" cd "$SCRIPT_DIR/.." -meson setup build --prefix `pwd`/../pginst --buildtype=$1 -Dcassert=true -Dtap_tests=enabled +meson setup build --prefix "$INSTALL_DIR" --buildtype="$1" -Dcassert=true -Dtap_tests=enabled $ENABLE_COVERAGE cd build && ninja && ninja install diff --git a/ci_scripts/meson-test-tde.sh b/ci_scripts/meson-test-global-tde.sh similarity index 100% rename from ci_scripts/meson-test-tde.sh rename to ci_scripts/meson-test-global-tde.sh diff --git a/ci_scripts/meson-test.sh b/ci_scripts/meson-test.sh index 5d0ad07e4ce48..f3f888d31c23f 100755 --- a/ci_scripts/meson-test.sh +++ b/ci_scripts/meson-test.sh @@ -1,7 +1,25 @@ #!/bin/bash +set -e +TDE_ONLY=0 + +for arg in "$@" +do + case "$arg" in + --tde-only) + TDE_ONLY=1 + shift;; + esac +done + SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)" +source "$SCRIPT_DIR/env.sh" cd "$SCRIPT_DIR/../build" -meson test +if [ "$TDE_ONLY" -eq 1 ]; +then + meson test --suite setup --suite pg_tde +else + meson test +fi diff --git a/ci_scripts/run-pgindent.sh b/ci_scripts/run-pgindent.sh index 13964ffe35c5c..4b9ba00f278b8 100755 --- a/ci_scripts/run-pgindent.sh +++ b/ci_scripts/run-pgindent.sh @@ -16,4 +16,8 @@ cd "$SCRIPT_DIR/.." export PATH=$SCRIPT_DIR/../src/tools/pgindent/:$INSTALL_DIR/bin/:$PATH -pgindent --typedefs=combined.typedefs "$@" . +# Check everything except pg_tde with the list in the repo +pgindent --typedefs=src/tools/pgindent/typedefs.list --excludes=<(echo "contrib/pg_tde") "$@" . + +# Check pg_tde with the fresh list extraxted from the object file +pgindent --typedefs=combined.typedefs "$@" contrib/pg_tde diff --git a/ci_scripts/run-pgperltidy.sh b/ci_scripts/run-pgperltidy.sh new file mode 100755 index 0000000000000..c59d50cc2a8f0 --- /dev/null +++ b/ci_scripts/run-pgperltidy.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)" +cd "$SCRIPT_DIR/../" + +source src/tools/perlcheck/find_perl_files + +find_perl_files contrib/pg_tde/ | xargs perltidy "$@" --profile=src/tools/pgindent/perltidyrc diff --git a/ci_scripts/tde_setup.sql b/ci_scripts/tde_setup.sql index 057119f86a5db..fd084c96462bd 100644 --- a/ci_scripts/tde_setup.sql +++ b/ci_scripts/tde_setup.sql @@ -1,4 +1,4 @@ CREATE SCHEMA IF NOT EXISTS tde; CREATE EXTENSION IF NOT EXISTS pg_tde SCHEMA tde; -SELECT tde.pg_tde_add_key_provider_file('reg_file-vault', '/tmp/pg_tde_test_keyring.per'); -SELECT tde.pg_tde_set_principal_key('test-db-principal-key', 'reg_file-vault'); +SELECT tde.pg_tde_add_database_key_provider_file('reg_file-vault', '/tmp/pg_tde_test_keyring.per'); +SELECT tde.pg_tde_set_key_using_database_key_provider('test-db-key', 'reg_file-vault'); diff --git a/ci_scripts/tde_setup_global.sql b/ci_scripts/tde_setup_global.sql index 5b4a9629a63e5..3ffcae4ad8250 100644 --- a/ci_scripts/tde_setup_global.sql +++ b/ci_scripts/tde_setup_global.sql @@ -2,8 +2,10 @@ CREATE SCHEMA tde; CREATE EXTENSION IF NOT EXISTS pg_tde SCHEMA tde; SELECT tde.pg_tde_add_global_key_provider_file('reg_file-global', '/tmp/pg_tde_test_keyring.per'); -SELECT tde.pg_tde_set_server_principal_key('global-principal-key', 'reg_file-global'); +SELECT tde.pg_tde_set_server_key_using_global_key_provider('server-key', 'reg_file-global'); ALTER SYSTEM SET pg_tde.wal_encrypt = on; ALTER SYSTEM SET default_table_access_method = 'tde_heap'; ALTER SYSTEM SET search_path = "$user",public,tde; +CHECKPOINT; +SELECT tde.pg_tde_set_default_key_using_global_key_provider('default-key', 'reg_file-global', false); -- restart required diff --git a/ci_scripts/ubuntu-deps.sh b/ci_scripts/ubuntu-deps.sh index 5cb62fb993212..f8028187dcfa7 100755 --- a/ci_scripts/ubuntu-deps.sh +++ b/ci_scripts/ubuntu-deps.sh @@ -42,6 +42,9 @@ DEPS=( # Test pg_tde python3-pykmip libhttp-server-simple-perl + lcov + # Run pgperltidy + perltidy ) sudo apt-get update diff --git a/contrib/pg_tde/.gitignore b/contrib/pg_tde/.gitignore index 4bc8580d6e00f..044af8f3c5cae 100644 --- a/contrib/pg_tde/.gitignore +++ b/contrib/pg_tde/.gitignore @@ -4,15 +4,16 @@ __pycache__ .vscode +/autom4te.cache /config.cache /config.log /config.status -/autom4te.cache /configure~ -/t/results +/log /results +/src/pg_tde_change_key_provider +/t/results /tmp_check -/log -# tools files -typedefs-full.list +# Tool files +/typedefs-full.list diff --git a/contrib/pg_tde/Makefile b/contrib/pg_tde/Makefile index 105f0523a4a77..d1dc561a80d58 100644 --- a/contrib/pg_tde/Makefile +++ b/contrib/pg_tde/Makefile @@ -5,33 +5,34 @@ MODULE_big = pg_tde EXTENSION = pg_tde DATA = pg_tde--1.0-rc.sql +# Since meson supports skipping test suites this is a make only feature +ifndef TDE_MODE REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/pg_tde/pg_tde.conf -# toast_descrypt needs to be the first test when running with pg_tde -# preinstalled and default_principal_key needs to run after key_provider. -REGRESS = toast_decrypt \ -access_control \ +# default_principal_key needs to run after key_provider. +REGRESS = access_control \ alter_index \ cache_alloc \ change_access_method \ insert_update_delete \ key_provider \ -keyprovider_dependency \ kmip_test \ +partition_table \ pg_tde_is_encrypted \ recreate_storage \ relocate \ -subtransaction \ tablespace \ +toast_decrypt \ vault_v2_test \ +version \ default_principal_key TAP_TESTS = 1 +endif OBJS = src/encryption/enc_tde.o \ src/encryption/enc_aes.o \ src/access/pg_tde_tdemap.o \ src/access/pg_tde_xlog.o \ -src/access/pg_tde_xlog_encrypt.o \ -src/transam/pg_tde_xact_handler.o \ +src/access/pg_tde_xlog_smgr.o \ src/keyring/keyring_curl.o \ src/keyring/keyring_file.o \ src/keyring/keyring_vault.o \ @@ -44,7 +45,6 @@ src/catalog/tde_principal_key.o \ src/common/pg_tde_shmem.o \ src/common/pg_tde_utils.o \ src/smgr/pg_tde_smgr.o \ -src/pg_tde_defs.o \ src/pg_tde_event_capture.o \ src/pg_tde_guc.o \ src/pg_tde.o \ @@ -55,6 +55,7 @@ src/libkmip/libkmip/src/kmip_memset.o SCRIPTS_built = src/pg_tde_change_key_provider +EXTRA_INSTALL += contrib/pg_buffercache contrib/test_decoding EXTRA_CLEAN += src/pg_tde_change_key_provider.o ifdef USE_PGXS diff --git a/contrib/pg_tde/Makefile.tools b/contrib/pg_tde/Makefile.tools index 0d28b8292fd20..4832c74c9db68 100644 --- a/contrib/pg_tde/Makefile.tools +++ b/contrib/pg_tde/Makefile.tools @@ -1,18 +1,18 @@ TDE_XLOG_OBJS = \ - src/access/pg_tde_xlog_encrypt.frontend + src/access/pg_tde_xlog_smgr.frontend TDE_OBJS = \ - src/access/pg_tde_tdemap.frontend \ - src/catalog/tde_keyring.frontend \ - src/catalog/tde_keyring_parse_opts.frontend \ - src/catalog/tde_principal_key.frontend \ - src/common/pg_tde_utils.frontend \ - src/encryption/enc_aes.frontend \ - src/encryption/enc_tde.frontend \ - src/keyring/keyring_api.frontend \ - src/keyring/keyring_curl.frontend \ - src/keyring/keyring_file.frontend \ - src/keyring/keyring_vault.frontend \ + src/access/pg_tde_tdemap.frontend \ + src/catalog/tde_keyring.frontend \ + src/catalog/tde_keyring_parse_opts.frontend \ + src/catalog/tde_principal_key.frontend \ + src/common/pg_tde_utils.frontend \ + src/encryption/enc_aes.frontend \ + src/encryption/enc_tde.frontend \ + src/keyring/keyring_api.frontend \ + src/keyring/keyring_curl.frontend \ + src/keyring/keyring_file.frontend \ + src/keyring/keyring_vault.frontend \ src/libkmip/libkmip/src/kmip.frontend \ src/libkmip/libkmip/src/kmip_bio.frontend \ src/libkmip/libkmip/src/kmip_locate.frontend \ @@ -20,7 +20,6 @@ TDE_OBJS = \ src/keyring/keyring_kmip.frontend \ src/keyring/keyring_kmip_impl.frontend - TDE_OBJS2 = $(TDE_OBJS:%=$(top_srcdir)/contrib/pg_tde/%) TDE_XLOG_OBJS2 = $(TDE_XLOG_OBJS:%=$(top_srcdir)/contrib/pg_tde/%) @@ -35,5 +34,4 @@ libtdexlog.a: $(TDE_XLOG_OBJS2) rm -f $@ $(AR) $(AROPT) $@ $^ - -tdelibs: libtde.a libtdexlog.a \ No newline at end of file +tdelibs: libtde.a libtdexlog.a diff --git a/contrib/pg_tde/README.md b/contrib/pg_tde/README.md index be689e7624fc8..ec367e2cdcab3 100644 --- a/contrib/pg_tde/README.md +++ b/contrib/pg_tde/README.md @@ -1,4 +1,5 @@ [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/percona/pg_tde/badge)](https://scorecard.dev/viewer/?uri=github.com/percona/pg_tde) +[![codecov](https://codecov.io/github/percona/postgres/graph/badge.svg?token=Wow78BMYdP)](https://codecov.io/github/percona/postgres) [![Forum](https://img.shields.io/badge/Forum-join-brightgreen)](https://forums.percona.com/) # pg_tde: Transparent Database Encryption for PostgreSQL @@ -27,11 +28,11 @@ Transparent Data Encryption offers encryption at the file level and solves the p ## Documentation -Full and comprehensive documentation about `pg_tde` is available at https://percona.github.io/pg_tde/. +Full and comprehensive documentation about `pg_tde` is available at https://docs.percona.com/pg-tde/index.html. ## Percona Server for PostgreSQL -Percona provides binary packages of `pg_tde` extension only for Percona Server for PostgreSQL. Learn how to install them or build `pg_tde` from sources for PSPG in the [documentation](https://percona.github.io/pg_tde/main/install.html). +Percona provides binary packages of `pg_tde` extension only for Percona Server for PostgreSQL. Learn how to install them or build `pg_tde` from sources for PSPG in the [documentation](https://docs.percona.com/pg-tde/install.html). ## Building from sources for community PostgreSQL 1. Install required dependencies (replace XX with 16 or 17) @@ -112,25 +113,25 @@ _See [Make Builds for Developers](https://github.com/percona/pg_tde/wiki/Make-bu ```sql -- For Vault-V2 key provider - -- pg_tde_add_key_provider_vault_v2(provider_name, vault_token, vault_url, vault_mount_path, vault_ca_path) - SELECT pg_tde_add_key_provider_vault_v2( + -- pg_tde_add_database_key_provider_vault_v2(provider_name, vault_token, vault_url, vault_mount_path, vault_ca_path) + SELECT pg_tde_add_database_key_provider_vault_v2( 'vault-provider', json_object( 'type' VALUE 'remote', 'url' VALUE 'http://localhost:8888/token' ), json_object( 'type' VALUE 'remote', 'url' VALUE 'http://localhost:8888/url' ), to_json('secret'::text), NULL); -- For File key provider - -- pg_tde_add_key_provider_file(provider_name, file_path); - SELECT pg_tde_add_key_provider_file('file','/tmp/pgkeyring'); + -- pg_tde_add_database_key_provider_file(provider_name, file_path); + SELECT pg_tde_add_database_key_provider_file('file','/tmp/pgkeyring'); ``` **Note: The `File` provided is intended for development and stores the keys unencrypted in the specified data file.** - 5. Set the principal key for the database using the `pg_tde_set_principal_key` function. + 5. Set the principal key for the database using the `pg_tde_set_key` function. ```sql - -- pg_tde_set_principal_key(principal_key_name, provider_name); - SELECT pg_tde_set_principal_key('my-principal-key','file'); + -- pg_tde_set_key_using_database_key_provider(key_name, provider_name); + SELECT pg_tde_set_key_using_database_key_provider('my-key','file'); ``` 6. Specify `tde_heap` access method during table creation diff --git a/contrib/pg_tde/documentation/docs/architecture.md b/contrib/pg_tde/documentation/docs/architecture.md new file mode 100644 index 0000000000000..f7cd70e4805a9 --- /dev/null +++ b/contrib/pg_tde/documentation/docs/architecture.md @@ -0,0 +1,384 @@ +# Architecture + +`pg_tde` is a **customizable, complete, data at rest encryption extension** for PostgreSQL. + +Let's break down what it means. + +**Customizable** means that `pg_tde` aims to support many different use cases: + +* Encrypting either every table in every database or only some tables in some databases +* Encryption keys can be stored on various external key storage servers including Hashicorp Vault, KMIP servers. +* Using one key for everything or different keys for different databases +* Storing every key on the same key storage, or using different storages for different databases +* Handling permissions: who can manage database specific or global permissions, who can create encrypted or not encrypted tables + +**Complete** means that `pg_tde` aims to encrypt data at rest. + +**Data at rest** means everything written to the disk. This includes the following: + +* Table data files +* Indexes +* Sequences +* Temporary tables +* Write Ahead Log (WAL) +* System tables (not yet implemented) +* Temporary files (not yet implemented) + +**Extension** means that `pg_tde` should be implemented only as an extension, possibly compatible with any PostgreSQL distribution, including the open source community version. This requires changes in the PostgreSQL core to make it more extensible. Therefore, `pg_tde` currently works only with the [Percona Server for PostgreSQL](https://docs.percona.com/postgresql/17/index.html) - a binary replacement of community PostgreSQL and included in Percona Distribution for PostgreSQL. + +## Main components + +The main components of `pg_tde` are the following: + +* **Core server changes** focus on making the server more extensible, allowing the main logic of `pg_tde` to remain separate, as an extension. Core changes also add encryption-awareness to some command line tools that have to work directly with encrypted tables or encrypted WAL files. + + [Percona Server for PostgreSQL location](https://github.com/percona/postgres/tree/{{tdebranch}}) + +* The **`pg_tde` extension itself** implements the encryption code by hooking into the extension points introduced in the core changes, and the already existing extension points in the PostgreSQL server. + + Everything is controllable with GUC variables and SQL statements, similar to other extensions. + +* The **keyring API / libraries** implement the key storage logic with different key providers. The API is internal only, the keyring libraries are part of the main library for simplicity. +In the future these could be extracted into separate shared libraries with an open API, allowing the use of third-party providers. + +## Encryption architecture + +### Two-key hierarchy + +`pg_tde` uses two kinds of keys for encryption: + +* Internal keys to encrypt the data. They are stored in PostgreSQL's data directory under `$PGDATA/pg_tde``. +* Higher-level keys to encrypt internal keys. These keys are called "principal keys". They are stored externally, in a Key Management System (KMS) using the key provider API. + +`pg_tde` uses one principal key per database. Every internal key for the given database is encrypted using this principal key. + +Internal keys are used for specific database files: each file with a different [Object Identifier (OID)](https://www.postgresql.org/docs/current/datatype-oid.html) has a different internal key. + +This means that, for example, a table with 4 indexes will have at least 5 internal keys - one for the table, and one for each index. + +If a table has additional associated relations, such as sequences or a TOAST table, those relations will also have separate keys. + +### Encryption algorithm + +`pg_tde` currently uses the following encryption algorithms: + +* `AES-128-CBC` for encrypting database files; encrypted with internal keys. + +* `AES-128-CTR` for WAL encryption; encrypted with internal keys. + +* `AES-128-GCM` for encrypting internal keys; encrypted with the principal key. + +Support for other cipher lengths / algorithms is planned in the future. + +### Encryption workflow + +`pg_tde` makes it possible to encrypt everything or only some tables in some databases. + +To support this without metadata changes, encrypted tables are labeled with a `tde_heap` access method marker. + +The `tde_heap` access method is the same as the `heap` one. It uses the same functions internally without any changes, but with the different name and ID. In such a way `pg_tde` knows that `tde_heap` tables are encrypted and `heap` tables are not. + +The initial decision what to encrypt is made using the `postgres` event trigger mechanism: if a `CREATE TABLE` or `ALTER TABLE` statement uses the `tde_heap` clause, the newly created data files are marked as encrypted. Then file operations encrypt or decrypt the data. + +Later decisions are made using a slightly modified Storage Manager (SMGR) API: when a database file is re-created with a different ID as a result of a `TRUNCATE` or a `VACUUM FULL` command, the newly created file inherits the encryption information and is either encrypted or not. + +### WAL encryption + +WAL encryption is controlled globally via a global GUC variable, `pg_tde.wal_encrypt`, that requires a server restart. + +WAL keys also contain the [LSN](https://www.postgresql.org/docs/17/wal-internals.html) of the first WAL write after key creation. This allows `pg_tde` to know which WAL ranges are encrypted or not and with which key. + +The setting only controls writes so that only WAL writes are encrypted when WAL encryption is enabled. This means that WAL files can contain both encrypted and unencrpyted data, depending on what the status of this variable was when writing the data. + +`pg_tde` keeps track of the encryption status of WAL records using internal keys. When the server is restarted it writes a new internal key if WAL encryption is enabled, or if it is disabled and was previously enabled it writes a dummy key signalling that WAL encryption ended. + +With this information the WAL reader code can decide if a specific WAL record has to be decrypted or not and which key it should use to decrypt it. + +### Encrypting other access methods + +Currently `pg_tde` only encrypts `heap` tables and other files such as indexes, TOAST tables, sequences that are related to the `heap` tables. + +Indexes include any kind of index that goes through the SMGR API, not just the built-in indexes in PostgreSQL. + +In theory, it is also possible to encrypt any other table access method that goes through the SMGR API by similarly providing a marker access method to it and extending the event triggers. + +### Storage Manager (SMGR) API + +`pg_tde` relies on a slightly modified version of the SMGR API. These modifications include: + +* Making the API generally extensible, where extensions can inject custom code into the storage manager +* Adding tracking information for files. When a new file is created for an existing relation, references to the existing file are also passed to the SMGR functions + +With these modifications, the `pg_tde` extension can implement an additional layer on top of the normal Magnetic Disk SMGR API: if the related table is encrypted, `pg_tde` encrypts a file before writing it to the disk and, similarly, decrypts it after reading when needed. + +## Key and key provider management + +### Principal key rotation + +You can rotate principal keys to comply with common policies and to handle situations with potentially exposed principal keys. + +Rotation means that `pg_tde` generates a new version of the principal key, and re-encrypts the associated internal keys with the new key. The old principal key is kept as is at the same location, because it may still be needed to decrypt backups or other databases. + +### Internal key regeneration + +Internal keys for tables, indexes and other data files are fixed once a file is created. There's no way to re-encrypt a file. + +There are workarounds for this, because operations that move the table data to a new file, such as `VACUUM FULL` or an `ALTER TABLE` that rewrites the file will create a new key for the new file, essentially rotating the internal key. This however means taking an exclusive lock on the table for the duration of the operation, which might not be desirable for huge tables. + +WAL internal keys are also fixed to the respective ranges. To generate a new WAL key you need to restart the database. + +### Internal key storage + +Internal keys and `pg_tde` metadata in general are kept in a single `$PGDATA/pg_tde` directory. This directory stores separate files for each database, such as: + +* Encrypted internal keys and internal key mapping to tables +* Information about the key providers + +Also, the `$PGDATA/pg_tde` directory has a special global section marked with the OID `1664`, which includes the global key providers and global internal keys. + +The global section is used for WAL encryption. Specific databases can use the global section too, for scenarios where users configure individual principal keys for databases but use the same global key provider. For this purpose, you must enable the global provider inheritance. + +The global default principal key uses the special OID `1663`. + +### Key providers (principal key storage) + +Principal keys are stored externally in a Key Management Services (KMS). In `pg_tde`a KMS is defined as an external key provider. + +The following key providers are supported: + +* [HashiCorp Vault](https://developer.hashicorp.com/vault/docs/what-is-vault) KV2 secrets engine +* [OpenBao](https://openbao.org/) implementation of Vault +* KMIP compatible servers +* A local file storage. This storage is intended only for development and testing and is not recommended for production use. + +For each key provider `pg_tde` requires a detailed configuration including the address of the service and the authentication information. + +With these details `pg_tde` does the following based on user operations: + +* Uploads a new principal key to it after this key is created +* Retrieves the principal key from the service when it is required for decryption + +Retreival of the principal key is cached so it only happens when necessary. + +### Key provider management + +Key provider configuration or location may change. For example, a service is moved to a new address or the principal key must be moved to a different key provider type. `pg_tde` supports both these scenarios enabling you to manage principal keys using simple [SQL functions](functions.md#key-provider-management). + +In certain cases you can't use SQL functions to manage key providers. For example, if the key provider changed while the server wasn't running and is therefore unaware of these changes. The startup can fail if it needs to access the encryption keys. + +For such situations, `pg_tde` also provides [command line tools](command-line-tools.md) to recover the database. + +### Sensitive key provider information + +Key provider information authentication details is a sensitive information. It is not safe to store it together with the database in the `$PGDATA` directory, or even on the same server. + +To safeguard key providers' sensitive information, `pg_tde` supports references to external services. Instead of specifying authentication details directly, users specify the reference to the external service where it is stored. `pg_tde` then downloads the provider's authentication details when needed. + +The currently supported external services are HTTP and external file references. + +## User interface + +### Setting up `pg_tde` + +To use `pg_tde`, users are required to: + +* Add `pg_tde` to the `shared_preload_libraries` in `postgresql.conf` as this is required for the SMGR extensions +* Execute `CREATE EXTENSION pg_tde` in the databases where they want to use encryption +* Optionally, enable `pg_tde.wal_encrypt` in `postgresql.conf` +* Optionally, disable `pg_tde.inherit_global_providers` in `postgresql.conf` (enabled by default) + +### Adding providers + +Keyring providers can be added to either the global or to the database specific scope. + +If `pg_tde.inherit_global_providers` is `on`, global providers are visible for all databases, and can be used. +If `pg_tde.inherit_global_providers` is `off`, global providers are only used for WAL encryption. + +To add a global provider: + +```sql +pg_tde_add_global_key_provider_('provider_name', ... details ...) +``` + +To add a database specific provider: + +```sql +pg_tde_add_database_key_provider_('provider_name', ... details ...) +``` + +### Changing providers + +To change a value of a global provider: + +```sql +pg_tde_change_global_key_provider_('provider_name', ... details ...) +``` + +To change a value of a database specific provider: + +```sql +pg_tde_change_database_key_provider_('provider_name', ... details ...) +``` + +These functions also allow changing the type of a provider. + +The functions however do not migrate any data. They are expected to be used during infrastructure migration, for example when the address of a server changes. + +Note that in these functions do not verify the parameters. For that, see `pg_tde_verify_key`. + +### Changing providers from the command line + +To change a provider from a command line, `pg_tde` provides the `pg_tde_change_key_provider` command line tool. + +This tool work similarly to the above functions, with the following syntax: + +```bash +pg_tde_change_key_provider ... details ... +``` + +Note that since this tool is expected to be offline, it bypasses all permission checks! + +This is also the reason why it requires a `dbOid` instead of a name, as it has no way to process the catalog and look up names. + +### Deleting providers + +Providers can be deleted by the + +```sql +pg_tde_delete_database_key_provider(provider_name) +pg_tde_delete_global_key_provider(provider_name) +``` + +functions. + +For database specific providers, the function first checks if the provider is used or not, and the provider is only deleted if it's not used. + +For global providers, the function checks if the provider is used anywhere, WAL or any specific database, and returns an error if it is. + +This somewhat goes against the principle that `pg_tde` shouldn't interact with other databases than the one the user is connected to, but on the other hand, it only does this lookup in the internal `pg_tde` metadata, not in postgres catalogs, so it is a gray zone. Making this check makes more sense than potentially making some databases inaccessible. + +### Listing/querying providers + +`pg_tde` provides 2 functions to show providers: + +* `pg_tde_list_all_database_key_providers()` +* `pg_tde_list_all_global_key_providers()` + +These functions return a list of provider names, type and configuration. + +### Provider permissions + +`pg_tde` implements access control based on execution rights on the administration functions. + +For keys and providers administration, it provides two pair of functions: + +```sql +pg_tde_(grant/revoke)_database_key_management_to_role +``` + +### Creating and rotating keys + +Principal keys can be created or rotated using the following functions: + +```sql +pg_tde_set_key_using_(global/database)_key_provider('key-name', 'provider-name', ensure_new_key) +pg_tde_set_server_key_using_(global/database)_key_provider('key-name', 'provider-name', ensure_new_key) +pg_tde_set_default_key_using_(global/database)_key_provider('key-name', 'provider-name', ensure_new_key) +``` + +`ensure_new_key` is a boolean parameter defaulting to false. If it is `true` the function might return an error instead of setting the key if it already exists on the provider. + +### Default principal key + +With `pg_tde.inherit_global_key_providers`, it is also possible to set up a default global principal key, which will be used by any database which has the `pg_tde` extension enabled, but doesn't have a database specific principal key configured using `pg_tde_set_key_using_(global/database)_key_provider`. + +With this feature, it is possible for the entire database server to easily use the same principal key for all databases, completely disabling multi-tenency. + +A default key can be managed with the following functions: + +```sql +pg_tde_set_default_key('key-name', 'provider-name', ensure_new_key) +pg_tde_drop_default_key() -- not yet implemented +``` + +`DROP` is only possible if there's no table currently using the default principal key. + +Changing the default principal key will rotate the encryption of internal keys for all databases using the current default principal key. + +### Removing key (not yet implemented) + +`pg_tde_drop_key` removes the principal key for the current database. If the current database has any encrypted tables, and there isn't a default principal key configured, it reports an error instead. If there are encrypted tables, but there's also a global default principal key, internal keys will be encrypted with the default key. + +It isn't possible to remove the WAL (server) principal key. + +### Current key details + +`pg_tde_key_info()` returns the name of the current principal key, and the provider it uses. + +`pg_tde_server_key_info()` does the same for the server key. + +`pg_tde_default_key_info()` does the same for the default key. + +`pg_tde_verify_key()` checks that the key provider is accessible, that the current principal key can be downloaded from it, and that it is the same as the current key stored in memory - if any of these fail, it reports an appropriate error. + +### Listing all active keys (not yet implemented) + +SUPERusers are able to use the following function: + +`pg_tde_list_active_keys()` + +Which reports all the actively used keys by all databases on the current server. Similarly to `pg_tde_key_info()`, it only shows names and associated providers, it doesn't reveal any sensitive information about the providers. + +### Key permissions + +Users with management permissions to a specific database `(pg_tde_(grant/revoke)_(global/databse)_key_management_to_role)` can change the keys for the database, and use the current key functions. This includes creating keys using global providers, if `pg_tde.inherit_global_providers` is enabled. + +Also the `pg_tde_(grant/revoke)_database_key_management_to_role` function deals with only the specific permission for the above function: it allows a user to change the key for the database, but not to modify the provider configuration. + +### Creating encrypted tables + +To create an encrypted table or modify an existing table to be encrypted, simply use `USING tde_heap` in the `CREATE` statement. + +### Changing the `pg_tde.inherit_global_keys` setting + +It is possible for users to use `pg_tde` with `inherit_global_keys = on`, refer to global keys / keyrings in databases, and then change this setting to `off`. + +In this case existing references to global providers, or the global default principal key will remain working as before, but new references to the global scope can't be made. + +## Typical setup scenarios + +### Simple "one principal key" encryption + +1. Installing the extension: `shared_preload_libraries` + `pg_tde.wal_encrypt` +2. `CREATE EXTENSION pg_tde;` in `template1` +3. Adding a global key provider +4. Adding a default principal key using the same global provider +5. Changing the WAL encryption to use the default principal key +6. Optionally: setting the `default_table_access_method` to `tde_heap` so that tables are encrypted by default + +Database users don't need permissions to any of the encryption functions: +encryption is managed by the admins, normal users only have to create tables with encryption, which requires no specific permissions. + +### One key storage, but different keys per database + +1. Installing the extension: `shared_preload_libraries` + `pg_tde.wal_encrypt` +2. `CREATE EXTENSION pg_tde;` in `template1` +3. Adding a global key provider +4. Changing the WAL encryption to use the proper global key provider +5. Giving users that are expected to manage database keys permissions for database specific key management, but not database specific key provider management: + specific databases HAVE to use the global key provider + +Note: setting the `default_table_access_method` to `tde_heap` is possible, but instead of `ALTER SYSTEM` only per database using `ALTER DATABASE`, after a principal key is configured for that specific database. + +Alternatively `ALTER SYSTEM` is possible, but table creation in the database will fail if there's no principal key for the database, that has to be created first. + +### Complete multi tenancy + +1. Installing the extension: `shared_preload_libraries` + `pg_tde.wal_encrypt` (that's not multi tenant currently) +2. `CREATE EXTENSION pg_tde;` in any database +2. Adding a global key provider for WAL +3. Changing the WAL encryption to use the proper global key provider + +No default configuration: key providers / principal keys are configured as a per database level, permissions are managed per database + +Same note about `default_table_access_method` as above - but in a multi tenant setup, `ALTER SYSTEM` doesn't make much sense. diff --git a/contrib/pg_tde/documentation/docs/command-line-tools.md b/contrib/pg_tde/documentation/docs/command-line-tools.md index e5b2763529bd4..a737505c4861c 100644 --- a/contrib/pg_tde/documentation/docs/command-line-tools.md +++ b/contrib/pg_tde/documentation/docs/command-line-tools.md @@ -12,11 +12,11 @@ Its only intended use is to fix servers that can't start up because of inaccessi For example, you restore from an old backup and the address of the key provider changed in the meantime. You can use this tool to correct the configuration, allowing the server to start up. -Use this tool **only when the server is offline.** To modify the key provider configuration when the server is up and running, use the [`pg_tde_change_key_provider_`](functions.md#change-an-existing-provider) SQL functions. +Use this tool **only when the server is offline.** To modify the key provider configuration when the server is up and running, use the [`pg_tde_change_(global/database)_key_provider_`](functions.md#change-an-existing-provider) SQL functions. ### Usage -To modify the key provider configuration, specify all parameters depending on the provider type in the same way as you do when using the [`pg_tde_change_key_provider_`](functions.md#change-an-existing-provider) SQL functions. +To modify the key provider configuration, specify all parameters depending on the provider type in the same way as you do when using the [`pg_tde_change_(global/database)_key_provider_`](functions.md#change-an-existing-provider) SQL functions. The general syntax is as follows: diff --git a/contrib/pg_tde/documentation/docs/external-parameters.md b/contrib/pg_tde/documentation/docs/external-parameters.md index a27e97b03127f..f68aee653e011 100644 --- a/contrib/pg_tde/documentation/docs/external-parameters.md +++ b/contrib/pg_tde/documentation/docs/external-parameters.md @@ -15,7 +15,7 @@ To use the file provider with a file location specified by the `remote` method, use the following command: ``` -SELECT pg_tde_add_key_provider_file( +SELECT pg_tde_add_database_key_provider_file( 'file-provider', json_object( 'type' VALUE 'remote', 'url' VALUE 'http://localhost:8888/hello' ) );" @@ -24,7 +24,7 @@ SELECT pg_tde_add_key_provider_file( Or to use the `file` method, use the following command: ``` -SELECT pg_tde_add_key_provider_file( +SELECT pg_tde_add_database_key_provider_file( 'file-provider', json_object( 'type' VALUE 'remote', 'path' VALUE '/tmp/datafile-location' ) );" diff --git a/contrib/pg_tde/documentation/docs/faq.md b/contrib/pg_tde/documentation/docs/faq.md index f7b951ba96218..a4842bdd306cd 100644 --- a/contrib/pg_tde/documentation/docs/faq.md +++ b/contrib/pg_tde/documentation/docs/faq.md @@ -79,9 +79,9 @@ Let's break the encryption into two parts: ### Encryption of data files -First, data files are encrypted with internal keys. Each file that has a different OID, has an internal key. For example, a table with 4 indexes will have 5 internal keys - one for the table and one for each index. +First, data files are encrypted with internal keys. Each file that has a different [Object Identifier (OID)](https://www.postgresql.org/docs/current/datatype-oid.html) has an internal key. For example, a table with 4 indexes will have 5 internal keys - one for the table and one for each index. -The initial decision on what file to encrypt is based on the table access method in PostgreSQL. When you run a `CREATE` or `ALTER TABLE` statement with the `USING tde_heap` clause, the newly created data files are marked as encrypted, and then file operations encrypt/decrypt the data. Later, if an initial file is re-created as a result of a `TRUNCATE` or `VACUUM FULL` command, the newly created file inherits the encryption information and is either encrypted or not. +The initial decision on what file to encrypt is based on the table access method in PostgreSQL. When you run a `CREATE` or `ALTER TABLE` statement with the `USING tde_heap` clause, the newly created data files are marked as encrypted, and then file operations encrypt or decrypt the data. Later, if an initial file is re-created as a result of a `TRUNCATE` or `VACUUM FULL` command, the newly created file inherits the encryption information and is either encrypted or not. The principal key is used to encrypt the internal keys. The principal key is stored in the key management store. When you query the table, the principal key is retrieved from the key store to decrypt the table. Then the internal key for that table is used to decrypt the data. @@ -91,7 +91,7 @@ WAL encryption is done globally for the entire database cluster. All modificatio When you turn on WAL encryption, `pg_tde` encrypts entire WAL files starting from the first WAL write after the server was started with the encryption turned on. -The same 2-tier approach is used with WAL as with the table data: WAL pages are first encrypted with the internal key. Then the internal key is encrypted with the global principal key. +The same 2-key approach is used with WAL as with the table data: WAL pages are first encrypted with the internal key. Then the internal key is encrypted with the global principal key. You can turn WAL encryption on and off so WAL can contain both encrypted and unencrypted data. The WAL encryption GUC variable influences only writes. diff --git a/contrib/pg_tde/documentation/docs/functions.md b/contrib/pg_tde/documentation/docs/functions.md index a59e1000bb6f7..4af66a4fb31be 100644 --- a/contrib/pg_tde/documentation/docs/functions.md +++ b/contrib/pg_tde/documentation/docs/functions.md @@ -4,7 +4,7 @@ The `pg_tde` extension provides functions for managing different aspects of its ## Permission management -By default, `pg_tde` is restrictive. It doesn't allow any operations until permissions are granted to the user. Only superusers can run permission management functions to manage user permissions. +By default, `pg_tde` is restrictive. It doesn't allow any operations until permissions are granted to the user. Only superusers can run permission management functions to manage user permissions. Operations on the global scope are limited to superusers only. Permissions are based on the normal `EXECUTE` permission on the functions provided by `pg_tde`. Superusers manage them using the `GRANT EXECUTE` and `REVOKE EXECUTE` commands. @@ -14,15 +14,12 @@ The following functions are also provided for easier management of functionality Use these functions to grant or revoke permissions to manage permissions for the current database. They enable or disable all functions related to the providers and keys on the current database: -* `pg_tde_grant_local_key_management_to_role(role)` -* `pg_tde_revoke_local_key_management_from_role(role)` +* `pg_tde_grant_database_key_management_to_role(role)` +* `pg_tde_revoke_database_key_management_from_role(role)` ### Global scope key management -Use these functions to grant or revoke permissions to manage permissions for the global scope - the entire PostgreSQL instance. They enable or disable all functions related to the providers and keys for the global scope: - -* `pg_tde_grant_global_key_management_to_role(role)` -* `pg_tde_revoke_global_key_management_from_role(role)` +Managment of the global scope is restricted to superusers only. ### Permission management @@ -61,7 +58,7 @@ You can add a new key provider using the provided functions, which are implement There are two functions to add a key provider: one function adds it for the current database and another one - for the global scope. -* `pg_tde_add_key_provider_('provider-name', )` +* `pg_tde_add_database_key_provider_('provider-name', )` * `pg_tde_add_global_key_provider_('provider-name', )` When you add a new provider, the provider name must be unqiue in the scope. But a local database provider and a global provider can have the same name. @@ -72,7 +69,7 @@ You can change an existing key provider using the provided functions, which are There are two functions to change existing providers: one to change a provider in the current database, and another one to change a provider in the global scope. -* `pg_tde_change_key_provider_('provider-name', )` +* `pg_tde_change_database_key_provider_('provider-name', )` * `pg_tde_change_global_key_provider_('provider-name', )` When you change a provider, the referred name must exist in the database local or a global scope. @@ -90,14 +87,14 @@ The Vault provider connects to a HashiCorp Vault or an OpenBao server, and store Use the following functions to add the Vault provider: ``` -SELECT pg_tde_add_key_provider_vault_v2('provider-name','secret_token','url','mount','ca_path'); +SELECT pg_tde_add_database_key_provider_vault_v2('provider-name','secret_token','url','mount','ca_path'); SELECT pg_tde_add_global_key_provider_vault_v2('provider-name','secret_token','url','mount','ca_path'); ``` These functions change the Vault provider: ``` -SELECT pg_tde_change_key_provider_vault_v2('provider-name','secret_token','url','mount','ca_path'); +SELECT pg_tde_change_database_key_provider_vault_v2('provider-name','secret_token','url','mount','ca_path'); SELECT pg_tde_change_global_key_provider_vault_v2('provider-name','secret_token','url','mount','ca_path'); ``` @@ -121,14 +118,14 @@ The KMIP provider uses a remote KMIP server. Use these functions to add a KMIP provider: ``` -SELECT pg_tde_add_key_provider_kmip('provider-name','kmip-addr', `port`, '/path_to/server_certificate.pem', '/path_to/client_key.pem'); +SELECT pg_tde_add_database_key_provider_kmip('provider-name','kmip-addr', `port`, '/path_to/server_certificate.pem', '/path_to/client_key.pem'); SELECT pg_tde_add_global_key_provider_kmip('provider-name','kmip-addr', `port`, '/path_to/server_certificate.pem', '/path_to/client_key.pem'); ``` These functions change the KMIP provider: ``` -SELECT pg_tde_change_key_provider_kmip('provider-name','kmip-addr', `port`, '/path_to/server_certificate.pem', '/path_to/client_key.pem'); +SELECT pg_tde_change_database_key_provider_kmip('provider-name','kmip-addr', `port`, '/path_to/server_certificate.pem', '/path_to/client_key.pem'); SELECT pg_tde_change_global_key_provider_kmip('provider-name','kmip-addr', `port`, '/path_to/server_certificate.pem', '/path_to/client_key.pem'); ``` @@ -156,14 +153,14 @@ This function is intended for development or quick testing, and stores the keys Add a local keyfile provider: ``` -SELECT pg_tde_add_key_provider_file('provider-name','/path/to/the/key/provider/data.file'); +SELECT pg_tde_add_database_key_provider_file('provider-name','/path/to/the/key/provider/data.file'); SELECT pg_tde_add_global_key_provider_file('provider-name','/path/to/the/key/provider/data.file'); ``` Change a local keyfile provider: ``` -SELECT pg_tde_change_key_provider_file('provider-name','/path/to/the/key/provider/data.file'); +SELECT pg_tde_change_database_key_provider_file('provider-name','/path/to/the/key/provider/data.file'); SELECT pg_tde_change_global_key_provider_file('provider-name','/path/to/the/key/provider/data.file'); ``` @@ -178,7 +175,7 @@ All parameters can be either strings, or JSON objects [referencing remote parame These functions delete an existing provider in the current database or in the global scope: -* `pg_tde_delete_key_provider('provider-name)` +* `pg_tde_delete_database_key_provider('provider-name)` * `pg_tde_delete_global_key_provider('provider-name)` You can only delete key providers that are not currently in use. An error is returned if the current principal key is using the provider you are trying to delete. @@ -189,7 +186,7 @@ If the use of global key providers is enabled via the `pg_tde.inherit_global` GU These functions list the details of all key providers for the current database or for the global scope, including all configuration values: -* `pg_tde_list_all_key_providers()` +* `pg_tde_list_all_database_key_providers()` * `pg_tde_list_all_global_key_providers()` **All configuration values include possibly sensitive values, such as passwords. Never specify these directly, use the remote configuration option instead.** @@ -201,12 +198,12 @@ Use these functions to create a new principal key for a specific scope such as a Princial keys are stored on key providers by the name specified in this function - for example, when using the Vault provider, after creating a key named "foo", a key named "foo" will be visible on the Vault server at the specified mount point. -### pg_tde_set_principal_key +### pg_tde_set_key_using_database_key_provider Creates or rotates the principal key for the current database using the specified database key provider and key name. ``` -SELECT pg_tde_set_principal_key('name-of-the-principal-key','provider-name','ensure_new_key'); +SELECT pg_tde_set_key_using_database_key_provider('name-of-the-key','provider-name','ensure_new_key'); ``` The `ensure_new_key` parameter instructs the function how to handle a principal key during key rotation: @@ -215,12 +212,12 @@ SELECT pg_tde_set_principal_key('name-of-the-principal-key','provider-name','ens If the provider already stores a key by that name, the function returns an error. * If set to `false`, an existing principal key may be reused. -### pg_tde_set_global_principal_key +### pg_tde_set_key_using_global_key_provider Creates or rotates the global principal key using the specified global key provider and the key name. This key is used for global settings like WAL encryption. ``` -SELECT pg_tde_set_global_principal_key('name-of-the-principal-key','provider-name','ensure_new_key'); +SELECT pg_tde_set_key_using_global_key_provider('name-of-the-key','provider-name','ensure_new_key'); ``` The `ensure_new_key` parameter instructs the function how to handle a principal key during key rotation: @@ -229,12 +226,12 @@ SELECT pg_tde_set_global_principal_key('name-of-the-principal-key','provider-nam If the provider already stores a key by that name, the function returns an error. * If set to `false`, an existing principal key may be reused. -### pg_tde_set_server_principal_key +### pg_tde_set_server_key_using_global_key_provider -Creates or rotates the global principal key using the specified key provider. Use this function to set a principal key for WAL encryption. +Creates or rotates the server principal key using the specified global key provider. Use this function to set a principal key for WAL encryption. ``` -SELECT pg_tde_set_server_principal_key('name-of-the-principal-key','provider-name','ensure_new_key'); +SELECT pg_tde_set_server_key_using_global_key_provider('name-of-the-key','provider-name','ensure_new_key'); ``` The `ensure_new_key` parameter instructs the function how to handle a principal key during key rotation: @@ -244,14 +241,14 @@ The `ensure_new_key` parameter instructs the function how to handle a principal * If set to `false`, an existing principal key may be reused. -### pg_tde_set_default_principal_key +### pg_tde_set_default_key_using_global_key_provider -Creates or rotates the default principal key for the server using the specified key provider. +Creates or rotates the default principal key for the server using the specified global key provider. The default key is automatically used as a principal key by any database that doesn't have an individual key provider and key configuration. ``` -SELECT pg_tde_set_default_principal_key('name-of-the-principal-key','provider-name','ensure_new_key'); +SELECT pg_tde_set_default_key_using_global_key_provider('name-of-the-key','provider-name','ensure_new_key'); ``` The `ensure_new_key` parameter instructs the function how to handle a principal key during key rotation: @@ -282,23 +279,31 @@ SELECT pg_tde_is_encrypted('schema.table_name'); This can additionally be used to verify that indexes and sequences are encrypted. -### pg_tde_principal_key_info +### pg_tde_key_info Displays information about the principal key for the current database, if it exists. ``` -SELECT pg_tde_principal_key_info() +SELECT pg_tde_key_info() ``` -### pg_tde_global_principal_key_info +### pg_tde_server_key_info -Displays information about the principal key for the global scope, if exists. +Displays information about the principal key for the server scope, if exists. ``` -SELECT pg_tde_global_principal_key_info() +SELECT pg_tde_server_key_info() ``` -### pg_tde_verify_principal_key +### pg_tde_default_key_info + +Displays the information about the default principal key, if it exists. + +``` +SELECT pg_tde_default_key_info() +``` + +### pg_tde_verify_key This function checks that the current database has a properly functional encryption setup, which means: @@ -311,12 +316,12 @@ This function checks that the current database has a properly functional encrypt If any of the above checks fail, the function reports an error. ``` -SELECT pg_tde_verify_principal_key() +SELECT pg_tde_verify_key() ``` -### pg_tde_verify_global_principal_key +### pg_tde_verify_server_key -This function checks that the global scope has a properly functional encryption setup, which means: +This function checks that the server scope has a properly functional encryption setup, which means: * A key provider is configured * The key provider is accessible using the specified configuration @@ -327,5 +332,21 @@ This function checks that the global scope has a properly functional encryption If any of the above checks fail, the function reports an error. ``` -SELECT pg_tde_verify_principal_key() +SELECT pg_tde_verify_server_key() +``` + +### pg_tde_verify_default_key + +This function checks that the default key is properly configured, which means: + +* A key provider is configured +* The key provider is accessible using the specified configuration +* There is a principal key that can be used for any scope +* The principal key can be retrieved from the remote key provider +* The principal key returned from the key provider is the same as cached in the server memory + +If any of the above checks fail, the function reports an error. + +``` +SELECT pg_tde_verify_default_key() ``` diff --git a/contrib/pg_tde/documentation/docs/index.md b/contrib/pg_tde/documentation/docs/index.md index 2a4095213bc1b..4bf8a3b9e0e7a 100644 --- a/contrib/pg_tde/documentation/docs/index.md +++ b/contrib/pg_tde/documentation/docs/index.md @@ -2,7 +2,7 @@ `pg_tde` is the open source PostgreSQL extension that provides Transparent Data Encryption (TDE) to protect data at rest. This ensures that the data stored on disk is encrypted, and no one can read it without the proper encryption keys, even if they gain access to the physical storage media. -Learn more [what is Transparent Data Encryption](tde.md#how-does-it-work) and [why you need it](tde.md#why-do-you-need-tde). +Learn more [what Transparent Data Encryption is](tde.md#how-does-it-work) and [why you need it](tde.md#why-do-you-need-tde). !!! important diff --git a/contrib/pg_tde/documentation/docs/multi-tenant-setup.md b/contrib/pg_tde/documentation/docs/multi-tenant-setup.md index 6d8b0f09300c6..5a6558208da43 100644 --- a/contrib/pg_tde/documentation/docs/multi-tenant-setup.md +++ b/contrib/pg_tde/documentation/docs/multi-tenant-setup.md @@ -61,7 +61,7 @@ You must do these steps for every database where you have created the extension. For testing purposes, you can use the PyKMIP server which enables you to set up required certificates. To use a real KMIP server, make sure to obtain the valid certificates issued by the key management appliance. ``` - SELECT pg_tde_add_key_provider_kmip('provider-name','kmip-addr', 5696, '/path_to/server_certificate.pem', '/path_to/client_key.pem'); + SELECT pg_tde_add_database_key_provider_kmip('provider-name','kmip-addr', 5696, '/path_to/server_certificate.pem', '/path_to/client_key.pem'); ``` where: @@ -75,7 +75,7 @@ You must do these steps for every database where you have created the extension. :material-information: Warning: This example is for testing purposes only: ``` - SELECT pg_tde_add_key_provider_kmip('kmip','127.0.0.1', 5696, '/tmp/server_certificate.pem', '/tmp/client_key_jane_doe.pem'); + SELECT pg_tde_add_database_key_provider_kmip('kmip','127.0.0.1', 5696, '/tmp/server_certificate.pem', '/tmp/client_key_jane_doe.pem'); ``` === "With HashiCorp Vault" @@ -83,20 +83,20 @@ You must do these steps for every database where you have created the extension. The Vault server setup is out of scope of this document. ```sql - SELECT pg_tde_add_key_provider_vault_v2('provider-name','root_token','url','mount','ca_path'); + SELECT pg_tde_add_database_key_provider_vault_v2('provider-name','secret_token','url','mount','ca_path'); ``` where: * `url` is the URL of the Vault server * `mount` is the mount point where the keyring should store the keys - * `root_token` is an access token with read and write access to the above mount point + * `secret_token` is an access token with read and write access to the above mount point * [optional] `ca_path` is the path of the CA file used for SSL verification :material-information: Warning: This example is for testing purposes only: ``` - SELECT pg_tde_add_key_provider_file_vault_v2('my-vault','http://vault.vault.svc.cluster.local:8200,'secret/data','hvs.zPuyktykA...example...ewUEnIRVaKoBzs2', NULL); + SELECT pg_tde_add_database_key_provider_file_vault_v2('my-vault','http://vault.vault.svc.cluster.local:8200,'secret/data','hvs.zPuyktykA...example...ewUEnIRVaKoBzs2', NULL); ``` === "With a keyring file" @@ -104,32 +104,32 @@ You must do these steps for every database where you have created the extension. This setup is intended for development and stores the keys unencrypted in the specified data file. ```sql - SELECT pg_tde_add_key_provider_file('provider-name','/path/to/the/keyring/data.file'); + SELECT pg_tde_add_database_key_provider_file('provider-name','/path/to/the/keyring/data.file'); ``` :material-information: Warning: This example is for testing purposes only: ```sql - SELECT pg_tde_add_key_provider_file('file-keyring','/tmp/pg_tde_test_local_keyring.per'); + SELECT pg_tde_add_database_key_provider_file('file-keyring','/tmp/pg_tde_test_local_keyring.per'); ``` 2. Add a principal key ```sql - SELECT pg_tde_set_principal_key('name-of-the-principal-key', 'provider-name','ensure_new_key'); + SELECT pg_tde_set_key_using_database_key_provider('name-of-the-key', 'provider-name','ensure_new_key'); ``` where: - * `name-of-the-principal-key` is the name of the principal key. You will use this name to identify the key. + * `name-of-the-key` is the name of the principal key. You will use this name to identify the key. * `provider-name` is the name of the key provider you added before. The principal key will be associated with this provider. * `ensure_new_key` defines if a principal key must be unique. The default value `true` means that you must speficy a unique key during key rotation. The `false` value allows reusing an existing principal key. :material-information: Warning: This example is for testing purposes only: ```sql - SELECT pg_tde_set_principal_key('test-db-master-key','file-vault','ensure_new_key'); + SELECT pg_tde_set_key_using_database_key_provider('test-db-master-key','file-vault','ensure_new_key'); ``` The key is auto-generated. diff --git a/contrib/pg_tde/documentation/docs/release-notes/rc.md b/contrib/pg_tde/documentation/docs/release-notes/rc.md index 9b07d45121f1d..9ee5fc16d73e5 100644 --- a/contrib/pg_tde/documentation/docs/release-notes/rc.md +++ b/contrib/pg_tde/documentation/docs/release-notes/rc.md @@ -39,20 +39,19 @@ This release provides the following features and improvements: ## Known issues -* The default `mlock` limit on Rocky Linux 8 for ARM64-based architectures is 64 Kb. The internal `pg_tde` key size is 40 bytes, which results in about 1600 keys that fall into the limit. When the `mlock` limit is reached, `pg_tde` cannot lock memory for more keys and can fail with the error. +* The default `mlock` limit on Rocky Linux 8 for ARM64-based architectures equals the memory page size and is 64 Kb. This results in the child process with `pg_tde` failing to allocate another memory page because the max memory limit is reached by the parent process. - To prevent this, you can change the `mlock` limit: + To prevent this, you can change the `mlock` limit to be at least twice bigger than the memory page size: - * temporarily for the current session using the `ulimit -l ` command. - * set a new hard limit in the `/etc/security/limits.conf` file. To do so, you require the superuser privileges. + * temporarily for the current session using the `ulimit -l ` command. + * set a new hard limit in the `/etc/security/limits.conf` file. To do so, you require the superuser privileges. - Adjust the limits with caution since it affects other processes running in your system. + Adjust the limits with caution since it affects other processes running in your system. * You can now delete global key providers even when their associated principal key is still in use. This known issue will be fixed in the next release. For now, avoid deleting global key providers. - ## Changelog ### New Features diff --git a/contrib/pg_tde/documentation/docs/setup.md b/contrib/pg_tde/documentation/docs/setup.md index 20c3514e50e20..9e8dbcb51b019 100644 --- a/contrib/pg_tde/documentation/docs/setup.md +++ b/contrib/pg_tde/documentation/docs/setup.md @@ -77,14 +77,14 @@ Load the `pg_tde` at startup time. The extension requires additional shared memo The Vault server setup is out of scope of this document. ``` - SELECT pg_tde_add_global_key_provider_vault_v2('provider-name','root_token','url','mount','ca_path'); + SELECT pg_tde_add_global_key_provider_vault_v2('provider-name','secret_token','url','mount','ca_path'); ``` where: * `url` is the URL of the Vault server * `mount` is the mount point where the keyring should store the keys - * `root_token` is an access token with read and write access to the above mount point + * `secret_token` is an access token with read and write access to the above mount point * [optional] `ca_path` is the path of the CA file used for SSL verification :material-information: Warning: This example is for testing purposes only: @@ -112,19 +112,19 @@ Load the `pg_tde` at startup time. The extension requires additional shared memo 2. Add a default principal key ```sql - SELECT pg_tde_set_default_principal_key('name-of-the-principal-key','provider-name','ensure_new_key'); + SELECT pg_tde_set_default_key_using_global_key_provider('name-of-the-key','provider-name','ensure_new_key'); ``` where: - * `name-of-the-principal-key` is the name of the principal key. You will use this name to identify the key. + * `name-of-the-key` is the name of the principal key. You will use this name to identify the key. * `provider-name` is the name of the key provider you added before. The principal key will be associated with this provider. * `ensure_new_key` defines if a principal key must be unique. The default value `true` means that you must speficy a unique key during key rotation. The `false` value allows reusing an existing principal key. :material-information: Warning: This example is for testing purposes only. Replace the key name and provider name with your values: ```sql - SELECT pg_tde_set_global_principal_key('test-db-master-key','file-vault','ensure_new_key'); + SELECT pg_tde_set_key_using_global_key_provider('test-db-master-key','file-vault','ensure_new_key'); ``` The key is auto-generated. diff --git a/contrib/pg_tde/documentation/docs/test.md b/contrib/pg_tde/documentation/docs/test.md index e848dd267faba..963ac360c1298 100644 --- a/contrib/pg_tde/documentation/docs/test.md +++ b/contrib/pg_tde/documentation/docs/test.md @@ -33,15 +33,7 @@ Here's how to do it: The function returns `t` if the table is encrypted and `f` - if not. -3. Rotate the principal key when needed: - - ``` - SELECT pg_tde_rotate_principal_key(); -- uses automatic key versionin - -- or - SELECT pg_tde_rotate_principal_key('new-principal-key', NULL); -- specify new key name - -- or - SELECT pg_tde_rotate_principal_key('new-principal-key', 'new-provider'); -- changeprovider - ``` +3. Rotate the principal key when needed, see [Principal key management](functions.md#principal-key-management)) ## Encrypt existing table diff --git a/contrib/pg_tde/documentation/docs/wal-encryption.md b/contrib/pg_tde/documentation/docs/wal-encryption.md index 9abdddd72d685..184ec61b3758c 100644 --- a/contrib/pg_tde/documentation/docs/wal-encryption.md +++ b/contrib/pg_tde/documentation/docs/wal-encryption.md @@ -32,7 +32,7 @@ Here's what to do: :material-information: Warning: This example is for testing purposes only: ``` - SELECT pg_tde_add_key_global_provider_kmip('kmip','127.0.0.1', 5696, '/tmp/server_certificate.pem', '/tmp/client_key_jane_doe.pem'); + SELECT pg_tde_add_key_using_global_key_provider_kmip('kmip','127.0.0.1', 5696, '/tmp/server_certificate.pem', '/tmp/client_key_jane_doe.pem'); ``` === "With HashiCorp Vault" @@ -61,7 +61,7 @@ Here's what to do: 3. Create principal key ```sql - SELECT pg_tde_set_server_principal_key('principal-key', 'provider-name'); + SELECT pg_tde_set_server_key_using_global_key_provider('key', 'provider-name'); ``` 4. Enable WAL level encryption using the `ALTER SYSTEM` command. You need the privileges of the superuser to run this command: diff --git a/contrib/pg_tde/documentation/mkdocs.yml b/contrib/pg_tde/documentation/mkdocs.yml index 3e03345be1daf..91ac69b6bfbe7 100644 --- a/contrib/pg_tde/documentation/mkdocs.yml +++ b/contrib/pg_tde/documentation/mkdocs.yml @@ -52,6 +52,7 @@ theme: - content.tabs.link - navigation.top - navigation.tracking + - navigation.indexes @@ -92,8 +93,6 @@ markdown_extensions: linenums: false - pymdownx.snippets: base_path: ["snippets"] -# auto_append: -# - services-banner.md - pymdownx.emoji: emoji_index: !!python/name:material.extensions.emoji.twemoji emoji_generator: !!python/name:material.extensions.emoji.to_svg @@ -171,6 +170,7 @@ nav: - Concepts: - "What is TDE": tde.md - table-access-method.md + - architecture.md - How to: - Set up multi-tenancy: multi-tenant-setup.md - Use reference to external parameters: external-parameters.md diff --git a/contrib/pg_tde/expected/access_control.out b/contrib/pg_tde/expected/access_control.out index e2df4fe87bb87..5186a6b117ce2 100644 --- a/contrib/pg_tde/expected/access_control.out +++ b/contrib/pg_tde/expected/access_control.out @@ -2,14 +2,42 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; CREATE USER regress_pg_tde_access_control; SET ROLE regress_pg_tde_access_control; -- should throw access denied -SELECT pg_tde_add_key_provider_file('file-vault', '/tmp/pg_tde_test_keyring.per'); -ERROR: permission denied for function pg_tde_add_key_provider_file -SELECT pg_tde_set_principal_key('test-db-principal-key', 'file-vault'); -ERROR: permission denied for function pg_tde_set_principal_key +SELECT pg_tde_add_database_key_provider_file('local-file-provider', '/tmp/pg_tde_test_keyring.per'); +ERROR: permission denied for function pg_tde_add_database_key_provider_file +SELECT pg_tde_set_key_using_database_key_provider('test-db-key', 'local-file-provider'); +ERROR: permission denied for function pg_tde_set_key_using_database_key_provider +SELECT pg_tde_add_global_key_provider_file('global-file-provider', '/tmp/pg_tde_test_keyring.per'); +ERROR: must be superuser to modify global key providers +SELECT pg_tde_set_key_using_global_key_provider('test-db-key', 'global-file-provider'); +ERROR: must be superuser to access global key providers +SELECT pg_tde_set_server_key_using_global_key_provider('wal-key','global-file-provider'); +ERROR: must be superuser to access global key providers +SELECT pg_tde_set_default_key_using_global_key_provider('def-key', 'global-file-provider'); +ERROR: must be superuser to access global key providers +SELECT pg_tde_delete_database_key_provider('local-file-provider'); +ERROR: permission denied for function pg_tde_delete_database_key_provider +SELECT pg_tde_delete_global_key_provider('global-file-provider'); +ERROR: must be superuser to modify global key providers +SELECT pg_tde_list_all_database_key_providers(); +ERROR: permission denied for function pg_tde_list_all_database_key_providers +SELECT pg_tde_list_all_global_key_providers(); +ERROR: permission denied for function pg_tde_list_all_global_key_providers +SELECT pg_tde_key_info(); +ERROR: permission denied for function pg_tde_key_info +SELECT pg_tde_server_key_info(); +ERROR: permission denied for function pg_tde_server_key_info +SELECT pg_tde_default_key_info(); +ERROR: permission denied for function pg_tde_default_key_info +SELECT pg_tde_verify_key(); +ERROR: permission denied for function pg_tde_verify_key +SELECT pg_tde_verify_server_key(); +ERROR: permission denied for function pg_tde_verify_server_key +SELECT pg_tde_verify_default_key(); +ERROR: permission denied for function pg_tde_verify_default_key RESET ROLE; -SELECT pg_tde_grant_local_key_management_to_role('regress_pg_tde_access_control'); - pg_tde_grant_local_key_management_to_role -------------------------------------------- +SELECT pg_tde_grant_database_key_management_to_role('regress_pg_tde_access_control'); + pg_tde_grant_database_key_management_to_role +---------------------------------------------- (1 row) @@ -21,37 +49,49 @@ SELECT pg_tde_grant_key_viewer_to_role('regress_pg_tde_access_control'); SET ROLE regress_pg_tde_access_control; -- should now be allowed -SELECT pg_tde_add_key_provider_file('file-vault', '/tmp/pg_tde_test_keyring.per'); - pg_tde_add_key_provider_file ------------------------------- - 1 +SELECT pg_tde_add_database_key_provider_file('local-file-provider', '/tmp/pg_tde_test_keyring.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 (1 row) -SELECT pg_tde_add_key_provider_file('file-2', '/tmp/pg_tde_test_keyring_2.per'); - pg_tde_add_key_provider_file ------------------------------- - 2 +SELECT pg_tde_set_key_using_database_key_provider('test-db-key', 'local-file-provider'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- + (1 row) -SELECT pg_tde_set_principal_key('test-db-principal-key', 'file-vault'); - pg_tde_set_principal_key --------------------------- - +SELECT * FROM pg_tde_list_all_database_key_providers(); + id | provider_name | provider_type | options +----+---------------------+---------------+------------------------------------------------------------ + 1 | local-file-provider | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring.per"} (1 row) -SELECT * FROM pg_tde_list_all_key_providers(); - id | provider_name | provider_type | options -----+---------------+---------------+-------------------------------------------------------------- - 1 | file-vault | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring.per"} - 2 | file-2 | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring_2.per"} -(2 rows) +SELECT key_name, key_provider_name, key_provider_id FROM pg_tde_key_info(); + key_name | key_provider_name | key_provider_id +-------------+---------------------+----------------- + test-db-key | local-file-provider | 1 +(1 row) -SELECT principal_key_name, key_provider_name, key_provider_id FROM pg_tde_principal_key_info(); - principal_key_name | key_provider_name | key_provider_id ------------------------+-------------------+----------------- - test-db-principal-key | file-vault | 1 +SELECT pg_tde_verify_key(); + pg_tde_verify_key +------------------- + (1 row) +-- only superuser +SELECT pg_tde_add_global_key_provider_file('global-file-provider', '/tmp/pg_tde_test_keyring.per'); +ERROR: must be superuser to modify global key providers +SELECT pg_tde_change_global_key_provider_file('global-file-provider', '/tmp/pg_tde_test_keyring.per'); +ERROR: must be superuser to modify global key providers +SELECT pg_tde_delete_global_key_provider('global-file-provider'); +ERROR: must be superuser to modify global key providers +SELECT pg_tde_set_key_using_global_key_provider('key1', 'global-file-provider'); +ERROR: must be superuser to access global key providers +SELECT pg_tde_set_default_key_using_global_key_provider('key1', 'global-file-provider'); +ERROR: must be superuser to access global key providers +SELECT pg_tde_set_server_key_using_global_key_provider('key1', 'global-file-provider'); +ERROR: must be superuser to access global key providers RESET ROLE; SELECT pg_tde_revoke_key_viewer_from_role('regress_pg_tde_access_control'); pg_tde_revoke_key_viewer_from_role @@ -61,9 +101,19 @@ SELECT pg_tde_revoke_key_viewer_from_role('regress_pg_tde_access_control'); SET ROLE regress_pg_tde_access_control; -- verify the view access is revoked -SELECT * FROM pg_tde_list_all_key_providers(); -ERROR: permission denied for function pg_tde_list_all_key_providers -SELECT principal_key_name, key_provider_name, key_provider_id FROM pg_tde_principal_key_info(); -ERROR: permission denied for function pg_tde_principal_key_info +SELECT * FROM pg_tde_list_all_database_key_providers(); +ERROR: permission denied for function pg_tde_list_all_database_key_providers +SELECT key_name, key_provider_name, key_provider_id FROM pg_tde_key_info(); +ERROR: permission denied for function pg_tde_key_info +SELECT pg_tde_verify_key(); +ERROR: permission denied for function pg_tde_verify_key +SELECT pg_tde_server_key_info(); +ERROR: permission denied for function pg_tde_server_key_info +SELECT pg_tde_default_key_info(); +ERROR: permission denied for function pg_tde_default_key_info +SELECT pg_tde_verify_server_key(); +ERROR: permission denied for function pg_tde_verify_server_key +SELECT pg_tde_verify_default_key(); +ERROR: permission denied for function pg_tde_verify_default_key RESET ROLE; DROP EXTENSION pg_tde CASCADE; diff --git a/contrib/pg_tde/expected/alter_index.out b/contrib/pg_tde/expected/alter_index.out index d1d343a448e1f..424c390204cc9 100644 --- a/contrib/pg_tde/expected/alter_index.out +++ b/contrib/pg_tde/expected/alter_index.out @@ -1,13 +1,13 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); - pg_tde_add_key_provider_file ------------------------------- - 1 +SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 (1 row) -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); - pg_tde_set_principal_key --------------------------- +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- (1 row) diff --git a/contrib/pg_tde/expected/cache_alloc.out b/contrib/pg_tde/expected/cache_alloc.out index 096d557f73614..215fa628deeec 100644 --- a/contrib/pg_tde/expected/cache_alloc.out +++ b/contrib/pg_tde/expected/cache_alloc.out @@ -1,14 +1,14 @@ -- Just checking there are no mem debug WARNINGs during the cache population CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); - pg_tde_add_key_provider_file ------------------------------- - 1 +SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 (1 row) -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); - pg_tde_set_principal_key --------------------------- +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- (1 row) diff --git a/contrib/pg_tde/expected/change_access_method.out b/contrib/pg_tde/expected/change_access_method.out index 1e66f894466d4..ce8f54af7067d 100644 --- a/contrib/pg_tde/expected/change_access_method.out +++ b/contrib/pg_tde/expected/change_access_method.out @@ -1,21 +1,21 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); - pg_tde_add_key_provider_file ------------------------------- - 1 +SELECT pg_tde_add_database_key_provider_file('file-vault', '/tmp/pg_tde_test_keyring.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 (1 row) -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); - pg_tde_set_principal_key --------------------------- +SELECT pg_tde_set_key_using_database_key_provider('test-db-key', 'file-vault'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- (1 row) CREATE TABLE country_table ( - country_id serial primary key, - country_name varchar(32) unique not null, - continent varchar(32) not null -) using tde_heap; + country_id serial primary key, + country_name varchar(32) unique not null, + continent varchar(32) not null +) USING tde_heap; INSERT INTO country_table (country_name, continent) VALUES ('Japan', 'Asia'), @@ -48,19 +48,7 @@ SELECT pg_tde_is_encrypted('country_table_pkey'); (1 row) -- Try changing the encrypted table to an unencrypted table -ALTER TABLE country_table SET access method heap; -SELECT pg_tde_is_encrypted('country_table_country_id_seq'); - pg_tde_is_encrypted ---------------------- - f -(1 row) - -SELECT pg_tde_is_encrypted('country_table_pkey'); - pg_tde_is_encrypted ---------------------- - f -(1 row) - +ALTER TABLE country_table SET ACCESS METHOD heap; -- Insert some more data INSERT INTO country_table (country_name, continent) VALUES ('France', 'Europe'), @@ -96,7 +84,7 @@ SELECT pg_tde_is_encrypted('country_table_pkey'); (1 row) -- Change it back to encrypted -ALTER TABLE country_table SET access method tde_heap; +ALTER TABLE country_table SET ACCESS METHOD tde_heap; INSERT INTO country_table (country_name, continent) VALUES ('China', 'Asia'), ('Brazil', 'South America'), @@ -133,36 +121,54 @@ SELECT pg_tde_is_encrypted('country_table_pkey'); t (1 row) +-- Test that we honor the default value +SET default_table_access_method = 'heap'; +ALTER TABLE country_table SET ACCESS METHOD DEFAULT; +SELECT pg_tde_is_encrypted('country_table'); + pg_tde_is_encrypted +--------------------- + f +(1 row) + +SET default_table_access_method = 'tde_heap'; +ALTER TABLE country_table SET ACCESS METHOD DEFAULT; +SELECT pg_tde_is_encrypted('country_table'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +RESET default_table_access_method; ALTER TABLE country_table ADD y text; -SELECT pg_tde_is_encrypted(('pg_toast.pg_toast_' || 'country_table'::regclass::oid)::regclass); +SELECT pg_tde_is_encrypted('pg_toast.pg_toast_' || 'country_table'::regclass::oid); pg_tde_is_encrypted --------------------- t (1 row) CREATE TABLE country_table2 ( - country_id serial primary key, - country_name text unique not null, - continent text not null + country_id serial primary key, + country_name text unique not null, + continent text not null ); -SET pg_tde.enforce_encryption = ON; +SET pg_tde.enforce_encryption = on; CREATE TABLE country_table3 ( - country_id serial primary key, - country_name text unique not null, - continent text not null + country_id serial primary key, + country_name text unique not null, + continent text not null ) USING heap; ERROR: pg_tde.enforce_encryption is ON, only the tde_heap access method is allowed. -ALTER TABLE country_table SET access method heap; +ALTER TABLE country_table SET ACCESS METHOD heap; ERROR: pg_tde.enforce_encryption is ON, only the tde_heap access method is allowed. -ALTER TABLE country_table2 SET access method tde_heap; +ALTER TABLE country_table2 SET ACCESS METHOD tde_heap; CREATE TABLE country_table3 ( - country_id serial primary key, - country_name text unique not null, - continent text not null -) using tde_heap; + country_id serial primary key, + country_name text unique not null, + continent text not null +) USING tde_heap; DROP TABLE country_table; DROP TABLE country_table2; DROP TABLE country_table3; -SET pg_tde.enforce_encryption = OFF; +SET pg_tde.enforce_encryption = off; DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/expected/default_principal_key.out b/contrib/pg_tde/expected/default_principal_key.out index ff23a39dea7d5..6dd46615419fe 100644 --- a/contrib/pg_tde/expected/default_principal_key.out +++ b/contrib/pg_tde/expected/default_principal_key.out @@ -1,30 +1,53 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_global_key_provider_file('file-provider','/tmp/pg_tde_regression_default_principal_key.per'); +CREATE EXTENSION IF NOT EXISTS pg_buffercache; +SELECT pg_tde_add_global_key_provider_file('file-provider','/tmp/pg_tde_regression_default_key.per'); pg_tde_add_global_key_provider_file ------------------------------------- -3 (1 row) -SELECT pg_tde_set_default_principal_key('default-principal-key', 'file-provider', false); - pg_tde_set_default_principal_key ----------------------------------- +-- Should fail: no default principal key for the server yet +SELECT pg_tde_verify_default_key(); +ERROR: principal key not configured for current database +-- Should fail: no default principal key for the server yet +SELECT key_provider_id, key_provider_name, key_name + FROM pg_tde_default_key_info(); +ERROR: Principal key does not exists for the database +HINT: Use set_key interface to set the principal key +SELECT pg_tde_set_default_key_using_global_key_provider('default-key', 'file-provider', false); + pg_tde_set_default_key_using_global_key_provider +-------------------------------------------------- + +(1 row) + +SELECT pg_tde_verify_default_key(); + pg_tde_verify_default_key +--------------------------- (1 row) +SELECT key_provider_id, key_provider_name, key_name + FROM pg_tde_default_key_info(); + key_provider_id | key_provider_name | key_name +-----------------+-------------------+------------- + -3 | file-provider | default-key +(1 row) + -- fails SELECT pg_tde_delete_global_key_provider('file-provider'); ERROR: Can't delete a provider which is currently in use SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); id | provider_name ----+--------------- + -1 | file-keyring -3 | file-provider -(1 row) +(2 rows) -- Should fail: no principal key for the database yet -SELECT key_provider_id, key_provider_name, principal_key_name - FROM pg_tde_principal_key_info(); +SELECT key_provider_id, key_provider_name, key_name + FROM pg_tde_key_info(); ERROR: Principal key does not exists for the database -HINT: Use set_principal_key interface to set the principal key +HINT: Use set_key interface to set the principal key -- Should succeed: "localizes" the default principal key for the database CREATE TABLE test_enc( @@ -34,11 +57,11 @@ CREATE TABLE test_enc( ) USING tde_heap; INSERT INTO test_enc (k) VALUES (1), (2), (3); -- Should succeed: create table localized the principal key -SELECT key_provider_id, key_provider_name, principal_key_name - FROM pg_tde_principal_key_info(); - key_provider_id | key_provider_name | principal_key_name ------------------+-------------------+----------------------- - -3 | file-provider | default-principal-key +SELECT key_provider_id, key_provider_name, key_name + FROM pg_tde_key_info(); + key_provider_id | key_provider_name | key_name +-----------------+-------------------+------------- + -3 | file-provider | default-key (1 row) SELECT current_database() AS regress_database @@ -46,11 +69,15 @@ SELECT current_database() AS regress_database CREATE DATABASE regress_pg_tde_other; \c regress_pg_tde_other CREATE EXTENSION pg_tde; --- Should fail: no principal key for the database yet -SELECT key_provider_id, key_provider_name, principal_key_name +CREATE EXTENSION pg_buffercache; +-- TODO +SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_principal_key_info(); -ERROR: Principal key does not exists for the database -HINT: Use set_principal_key interface to set the principal key + key_provider_id | key_provider_name | key_name +-----------------+-------------------+------------- + -3 | file-provider | default-key +(1 row) + -- Should succeed: "localizes" the default principal key for the database CREATE TABLE test_enc( id SERIAL, @@ -59,38 +86,68 @@ CREATE TABLE test_enc( ) USING tde_heap; INSERT INTO test_enc (k) VALUES (1), (2), (3); -- Should succeed: create table localized the principal key -SELECT key_provider_id, key_provider_name, principal_key_name - FROM pg_tde_principal_key_info(); - key_provider_id | key_provider_name | principal_key_name ------------------+-------------------+----------------------- - -3 | file-provider | default-principal-key +SELECT key_provider_id, key_provider_name, key_name + FROM pg_tde_key_info(); + key_provider_id | key_provider_name | key_name +-----------------+-------------------+------------- + -3 | file-provider | default-key (1 row) \c :regress_database -SELECT pg_tde_set_default_principal_key('new-default-principal-key', 'file-provider', false); - pg_tde_set_default_principal_key ----------------------------------- +CHECKPOINT; +SELECT pg_tde_set_default_key_using_global_key_provider('new-default-key', 'file-provider', false); + pg_tde_set_default_key_using_global_key_provider +-------------------------------------------------- (1 row) -SELECT key_provider_id, key_provider_name, principal_key_name - FROM pg_tde_principal_key_info(); - key_provider_id | key_provider_name | principal_key_name ------------------+-------------------+--------------------------- - -3 | file-provider | new-default-principal-key +SELECT key_provider_id, key_provider_name, key_name + FROM pg_tde_key_info(); + key_provider_id | key_provider_name | key_name +-----------------+-------------------+----------------- + -3 | file-provider | new-default-key (1 row) \c regress_pg_tde_other -SELECT key_provider_id, key_provider_name, principal_key_name - FROM pg_tde_principal_key_info(); - key_provider_id | key_provider_name | principal_key_name ------------------+-------------------+--------------------------- - -3 | file-provider | new-default-principal-key +SELECT key_provider_id, key_provider_name, key_name + FROM pg_tde_key_info(); + key_provider_id | key_provider_name | key_name +-----------------+-------------------+----------------- + -3 | file-provider | new-default-key (1 row) +SELECT pg_buffercache_evict(bufferid) FROM pg_buffercache WHERE relfilenode = (SELECT relfilenode FROM pg_class WHERE oid = 'test_enc'::regclass); + pg_buffercache_evict +---------------------- + t +(1 row) + +SELECT * FROM test_enc; + id | k +----+--- + 1 | 1 + 2 | 2 + 3 | 3 +(3 rows) + DROP TABLE test_enc; DROP EXTENSION pg_tde CASCADE; \c :regress_database +SELECT pg_buffercache_evict(bufferid) FROM pg_buffercache WHERE relfilenode = (SELECT relfilenode FROM pg_class WHERE oid = 'test_enc'::regclass); + pg_buffercache_evict +---------------------- + t +(1 row) + +SELECT * FROM test_enc; + id | k +----+--- + 1 | 1 + 2 | 2 + 3 | 3 +(3 rows) + DROP TABLE test_enc; DROP EXTENSION pg_tde CASCADE; +DROP EXTENSION pg_buffercache; DROP DATABASE regress_pg_tde_other; diff --git a/contrib/pg_tde/expected/default_principal_key_1.out b/contrib/pg_tde/expected/default_principal_key_1.out deleted file mode 100644 index f8a6b17056c33..0000000000000 --- a/contrib/pg_tde/expected/default_principal_key_1.out +++ /dev/null @@ -1,97 +0,0 @@ -CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_global_key_provider_file('file-provider','/tmp/pg_tde_regression_default_principal_key.per'); - pg_tde_add_global_key_provider_file -------------------------------------- - -4 -(1 row) - -SELECT pg_tde_set_default_principal_key('default-principal-key', 'file-provider', false); - pg_tde_set_default_principal_key ----------------------------------- - -(1 row) - --- fails -SELECT pg_tde_delete_global_key_provider('file-provider'); -ERROR: Can't delete a provider which is currently in use -SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); - id | provider_name -----+----------------- - -1 | reg_file-global - -4 | file-provider -(2 rows) - --- Should fail: no principal key for the database yet -SELECT key_provider_id, key_provider_name, principal_key_name - FROM pg_tde_principal_key_info(); -ERROR: Principal key does not exists for the database -HINT: Use set_principal_key interface to set the principal key - --- Should succeed: "localizes" the default principal key for the database -CREATE TABLE test_enc( - id SERIAL, - k INTEGER DEFAULT '0' NOT NULL, - PRIMARY KEY (id) -) USING tde_heap; -INSERT INTO test_enc (k) VALUES (1), (2), (3); --- Should succeed: create table localized the principal key -SELECT key_provider_id, key_provider_name, principal_key_name - FROM pg_tde_principal_key_info(); - key_provider_id | key_provider_name | principal_key_name ------------------+-------------------+----------------------- - -4 | file-provider | default-principal-key -(1 row) - -SELECT current_database() AS regress_database -\gset -CREATE DATABASE regress_pg_tde_other; -\c regress_pg_tde_other -CREATE EXTENSION pg_tde; --- Should fail: no principal key for the database yet -SELECT key_provider_id, key_provider_name, principal_key_name - FROM pg_tde_principal_key_info(); -ERROR: Principal key does not exists for the database -HINT: Use set_principal_key interface to set the principal key --- Should succeed: "localizes" the default principal key for the database -CREATE TABLE test_enc( - id SERIAL, - k INTEGER DEFAULT '0' NOT NULL, - PRIMARY KEY (id) -) USING tde_heap; -INSERT INTO test_enc (k) VALUES (1), (2), (3); --- Should succeed: create table localized the principal key -SELECT key_provider_id, key_provider_name, principal_key_name - FROM pg_tde_principal_key_info(); - key_provider_id | key_provider_name | principal_key_name ------------------+-------------------+----------------------- - -4 | file-provider | default-principal-key -(1 row) - -\c :regress_database -SELECT pg_tde_set_default_principal_key('new-default-principal-key', 'file-provider', false); - pg_tde_set_default_principal_key ----------------------------------- - -(1 row) - -SELECT key_provider_id, key_provider_name, principal_key_name - FROM pg_tde_principal_key_info(); - key_provider_id | key_provider_name | principal_key_name ------------------+-------------------+--------------------------- - -4 | file-provider | new-default-principal-key -(1 row) - -\c regress_pg_tde_other -SELECT key_provider_id, key_provider_name, principal_key_name - FROM pg_tde_principal_key_info(); - key_provider_id | key_provider_name | principal_key_name ------------------+-------------------+--------------------------- - -4 | file-provider | new-default-principal-key -(1 row) - -DROP TABLE test_enc; -DROP EXTENSION pg_tde CASCADE; -\c :regress_database -DROP TABLE test_enc; -DROP EXTENSION pg_tde CASCADE; -DROP DATABASE regress_pg_tde_other; diff --git a/contrib/pg_tde/expected/delete_key_provider.out b/contrib/pg_tde/expected/delete_key_provider.out deleted file mode 100644 index f4f4ed109db53..0000000000000 --- a/contrib/pg_tde/expected/delete_key_provider.out +++ /dev/null @@ -1,74 +0,0 @@ -CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT * FROM pg_tde_principal_key_info(); -ERROR: Principal key does not exists for the database -HINT: Use set_principal_key interface to set the principal key -SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per'); - pg_tde_add_key_provider_file ------------------------------- - 1 -(1 row) - -SELECT * FROM pg_tde_list_all_key_providers(); - id | provider_name | provider_type | options -----+---------------+---------------+------------------------------------------------------------ - 1 | file-provider | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring.per"} -(1 row) - -SELECT pg_tde_delete_key_provider('file-provider'); - pg_tde_delete_key_provider ----------------------------- - -(1 row) - -SELECT * FROM pg_tde_list_all_key_providers(); - id | provider_name | provider_type | options -----+---------------+---------------+--------- -(0 rows) - -SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per'); - pg_tde_add_key_provider_file ------------------------------- - 2 -(1 row) - -SELECT * FROM pg_tde_list_all_key_providers(); - id | provider_name | provider_type | options -----+---------------+---------------+------------------------------------------------------------ - 2 | file-provider | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring.per"} -(1 row) - -SELECT pg_tde_delete_key_provider('file-provider'); - pg_tde_delete_key_provider ----------------------------- - -(1 row) - -SELECT * FROM pg_tde_list_all_key_providers(); - id | provider_name | provider_type | options -----+---------------+---------------+--------- -(0 rows) - -SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per'); - pg_tde_add_key_provider_file ------------------------------- - 3 -(1 row) - -SELECT * FROM pg_tde_list_all_key_providers(); - id | provider_name | provider_type | options -----+---------------+---------------+------------------------------------------------------------ - 3 | file-provider | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring.per"} -(1 row) - -SELECT pg_tde_delete_key_provider('file-provider'); - pg_tde_delete_key_provider ----------------------------- - -(1 row) - -SELECT * FROM pg_tde_list_all_key_providers(); - id | provider_name | provider_type | options -----+---------------+---------------+--------- -(0 rows) - -DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/expected/insert_update_delete.out b/contrib/pg_tde/expected/insert_update_delete.out index f9ac74fbf988c..144c5bc49e6ad 100644 --- a/contrib/pg_tde/expected/insert_update_delete.out +++ b/contrib/pg_tde/expected/insert_update_delete.out @@ -1,13 +1,13 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); - pg_tde_add_key_provider_file ------------------------------- - 1 +SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 (1 row) -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); - pg_tde_set_principal_key --------------------------- +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- (1 row) diff --git a/contrib/pg_tde/expected/key_provider.out b/contrib/pg_tde/expected/key_provider.out index 47930b72efe5f..51f45f3eb2200 100644 --- a/contrib/pg_tde/expected/key_provider.out +++ b/contrib/pg_tde/expected/key_provider.out @@ -1,81 +1,81 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT * FROM pg_tde_principal_key_info(); +SELECT * FROM pg_tde_key_info(); ERROR: Principal key does not exists for the database -HINT: Use set_principal_key interface to set the principal key -SELECT pg_tde_add_key_provider_file('incorrect-file-provider', json_object('foo' VALUE '/tmp/pg_tde_test_keyring.per')); +HINT: Use set_key interface to set the principal key +SELECT pg_tde_add_database_key_provider_file('incorrect-file-provider', json_object('foo' VALUE '/tmp/pg_tde_test_keyring.per')); ERROR: parse json keyring config: unexpected field foo -SELECT * FROM pg_tde_list_all_key_providers(); +SELECT * FROM pg_tde_list_all_database_key_providers(); id | provider_name | provider_type | options ----+---------------+---------------+--------- (0 rows) -SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per'); - pg_tde_add_key_provider_file ------------------------------- - 1 +SELECT pg_tde_add_database_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 (1 row) -SELECT * FROM pg_tde_list_all_key_providers(); +SELECT * FROM pg_tde_list_all_database_key_providers(); id | provider_name | provider_type | options ----+---------------+---------------+------------------------------------------------------------ 1 | file-provider | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring.per"} (1 row) -SELECT pg_tde_add_key_provider_file('file-provider2','/tmp/pg_tde_test_keyring2.per'); - pg_tde_add_key_provider_file ------------------------------- - 2 +SELECT pg_tde_add_database_key_provider_file('file-provider2','/tmp/pg_tde_test_keyring2.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 2 (1 row) -SELECT * FROM pg_tde_list_all_key_providers(); +SELECT * FROM pg_tde_list_all_database_key_providers(); id | provider_name | provider_type | options ----+----------------+---------------+------------------------------------------------------------- 1 | file-provider | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring.per"} 2 | file-provider2 | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring2.per"} (2 rows) -SELECT pg_tde_verify_principal_key(); +SELECT pg_tde_verify_key(); ERROR: principal key not configured for current database -SELECT pg_tde_set_principal_key('test-db-principal-key','file-provider'); - pg_tde_set_principal_key --------------------------- +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-provider'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- (1 row) -SELECT pg_tde_verify_principal_key(); - pg_tde_verify_principal_key ------------------------------ +SELECT pg_tde_verify_key(); + pg_tde_verify_key +------------------- (1 row) -SELECT pg_tde_change_key_provider_file('not-existent-provider','/tmp/pg_tde_test_keyring.per'); +SELECT pg_tde_change_database_key_provider_file('not-existent-provider','/tmp/pg_tde_test_keyring.per'); ERROR: key provider "not-existent-provider" does not exists -HINT: Use pg_tde_add_key_provider interface to create the key provider -SELECT * FROM pg_tde_list_all_key_providers(); +HINT: Create the key provider +SELECT * FROM pg_tde_list_all_database_key_providers(); id | provider_name | provider_type | options ----+----------------+---------------+------------------------------------------------------------- 1 | file-provider | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring.per"} 2 | file-provider2 | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring2.per"} (2 rows) -SELECT pg_tde_change_key_provider_file('file-provider','/tmp/pg_tde_test_keyring_other.per'); - pg_tde_change_key_provider_file ---------------------------------- - 1 +SELECT pg_tde_change_database_key_provider_file('file-provider','/tmp/pg_tde_test_keyring_other.per'); + pg_tde_change_database_key_provider_file +------------------------------------------ + 1 (1 row) -SELECT * FROM pg_tde_list_all_key_providers(); +SELECT * FROM pg_tde_list_all_database_key_providers(); id | provider_name | provider_type | options ----+----------------+---------------+------------------------------------------------------------------ 1 | file-provider | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring_other.per"} 2 | file-provider2 | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring2.per"} (2 rows) -SELECT pg_tde_verify_principal_key(); -ERROR: failed to retrieve principal key test-db-principal-key from keyring with ID 1 -SELECT pg_tde_change_key_provider_file('file-provider', json_object('foo' VALUE '/tmp/pg_tde_test_keyring.per')); +SELECT pg_tde_verify_key(); +ERROR: failed to retrieve principal key test-db-key from keyring with ID 1 +SELECT pg_tde_change_database_key_provider_file('file-provider', json_object('foo' VALUE '/tmp/pg_tde_test_keyring.per')); ERROR: parse json keyring config: unexpected field foo -SELECT * FROM pg_tde_list_all_key_providers(); +SELECT * FROM pg_tde_list_all_database_key_providers(); id | provider_name | provider_type | options ----+----------------+---------------+------------------------------------------------------------------ 1 | file-provider | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring_other.per"} @@ -101,11 +101,10 @@ SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); -2 | file-keyring2 (2 rows) --- TODO: verify that we can also can change the type of it -- fails -SELECT pg_tde_delete_key_provider('file-provider'); +SELECT pg_tde_delete_database_key_provider('file-provider'); ERROR: Can't delete a provider which is currently in use -SELECT id, provider_name FROM pg_tde_list_all_key_providers(); +SELECT id, provider_name FROM pg_tde_list_all_database_key_providers(); id | provider_name ----+---------------- 1 | file-provider @@ -113,13 +112,13 @@ SELECT id, provider_name FROM pg_tde_list_all_key_providers(); (2 rows) -- works -SELECT pg_tde_delete_key_provider('file-provider2'); - pg_tde_delete_key_provider ----------------------------- +SELECT pg_tde_delete_database_key_provider('file-provider2'); + pg_tde_delete_database_key_provider +------------------------------------- (1 row) -SELECT id, provider_name FROM pg_tde_list_all_key_providers(); +SELECT id, provider_name FROM pg_tde_list_all_database_key_providers(); id | provider_name ----+--------------- 1 | file-provider @@ -132,24 +131,21 @@ SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); -2 | file-keyring2 (2 rows) -SELECT pg_tde_set_global_principal_key('test-db-principal-key', 'file-keyring', false); - pg_tde_set_global_principal_key ---------------------------------- +SELECT pg_tde_set_key_using_global_key_provider('test-db-key', 'file-keyring', false); + pg_tde_set_key_using_global_key_provider +------------------------------------------ (1 row) -- fails SELECT pg_tde_delete_global_key_provider('file-keyring'); - pg_tde_delete_global_key_provider ------------------------------------ - -(1 row) - +ERROR: Can't delete a provider which is currently in use SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); id | provider_name ----+--------------- + -1 | file-keyring -2 | file-keyring2 -(1 row) +(2 rows) -- works SELECT pg_tde_delete_global_key_provider('file-keyring2'); @@ -161,6 +157,10 @@ SELECT pg_tde_delete_global_key_provider('file-keyring2'); SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); id | provider_name ----+--------------- -(0 rows) + -1 | file-keyring +(1 row) +-- Creating a file key provider fails if we can't open or create the file +SELECT pg_tde_add_database_key_provider_file('will-not-work','/cant-create-file-in-root.per'); +ERROR: Failed to open keyring file /cant-create-file-in-root.per: Permission denied DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/expected/key_provider_1.out b/contrib/pg_tde/expected/key_provider_1.out deleted file mode 100644 index 18ba31455eba3..0000000000000 --- a/contrib/pg_tde/expected/key_provider_1.out +++ /dev/null @@ -1,170 +0,0 @@ -CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT * FROM pg_tde_principal_key_info(); -ERROR: Principal key does not exists for the database -HINT: Use set_principal_key interface to set the principal key -SELECT pg_tde_add_key_provider_file('incorrect-file-provider', json_object('foo' VALUE '/tmp/pg_tde_test_keyring.per')); -ERROR: parse json keyring config: unexpected field foo -SELECT * FROM pg_tde_list_all_key_providers(); - id | provider_name | provider_type | options -----+---------------+---------------+--------- -(0 rows) - -SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per'); - pg_tde_add_key_provider_file ------------------------------- - 1 -(1 row) - -SELECT * FROM pg_tde_list_all_key_providers(); - id | provider_name | provider_type | options -----+---------------+---------------+------------------------------------------------------------ - 1 | file-provider | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring.per"} -(1 row) - -SELECT pg_tde_add_key_provider_file('file-provider2','/tmp/pg_tde_test_keyring2.per'); - pg_tde_add_key_provider_file ------------------------------- - 2 -(1 row) - -SELECT * FROM pg_tde_list_all_key_providers(); - id | provider_name | provider_type | options -----+----------------+---------------+------------------------------------------------------------- - 1 | file-provider | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring.per"} - 2 | file-provider2 | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring2.per"} -(2 rows) - -SELECT pg_tde_verify_principal_key(); -ERROR: principal key not configured for current database -SELECT pg_tde_set_principal_key('test-db-principal-key','file-provider'); - pg_tde_set_principal_key --------------------------- - -(1 row) - -SELECT pg_tde_verify_principal_key(); - pg_tde_verify_principal_key ------------------------------ - -(1 row) - -SELECT pg_tde_change_key_provider_file('not-existent-provider','/tmp/pg_tde_test_keyring.per'); -ERROR: key provider "not-existent-provider" does not exists -HINT: Use pg_tde_add_key_provider interface to create the key provider -SELECT * FROM pg_tde_list_all_key_providers(); - id | provider_name | provider_type | options -----+----------------+---------------+------------------------------------------------------------- - 1 | file-provider | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring.per"} - 2 | file-provider2 | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring2.per"} -(2 rows) - -SELECT pg_tde_change_key_provider_file('file-provider','/tmp/pg_tde_test_keyring_other.per'); - pg_tde_change_key_provider_file ---------------------------------- - 1 -(1 row) - -SELECT * FROM pg_tde_list_all_key_providers(); - id | provider_name | provider_type | options -----+----------------+---------------+------------------------------------------------------------------ - 1 | file-provider | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring_other.per"} - 2 | file-provider2 | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring2.per"} -(2 rows) - -SELECT pg_tde_verify_principal_key(); -ERROR: failed to retrieve principal key test-db-principal-key from keyring with ID 1 -SELECT pg_tde_change_key_provider_file('file-provider', json_object('foo' VALUE '/tmp/pg_tde_test_keyring.per')); -ERROR: parse json keyring config: unexpected field foo -SELECT * FROM pg_tde_list_all_key_providers(); - id | provider_name | provider_type | options -----+----------------+---------------+------------------------------------------------------------------ - 1 | file-provider | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring_other.per"} - 2 | file-provider2 | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring2.per"} -(2 rows) - -SELECT pg_tde_add_global_key_provider_file('file-keyring','/tmp/pg_tde_test_keyring.per'); - pg_tde_add_global_key_provider_file -------------------------------------- - -2 -(1 row) - -SELECT pg_tde_add_global_key_provider_file('file-keyring2','/tmp/pg_tde_test_keyring2.per'); - pg_tde_add_global_key_provider_file -------------------------------------- - -3 -(1 row) - -SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); - id | provider_name -----+----------------- - -1 | reg_file-global - -2 | file-keyring - -3 | file-keyring2 -(3 rows) - --- TODO: verify that we can also can change the type of it --- fails -SELECT pg_tde_delete_key_provider('file-provider'); -ERROR: Can't delete a provider which is currently in use -SELECT id, provider_name FROM pg_tde_list_all_key_providers(); - id | provider_name -----+---------------- - 1 | file-provider - 2 | file-provider2 -(2 rows) - --- works -SELECT pg_tde_delete_key_provider('file-provider2'); - pg_tde_delete_key_provider ----------------------------- - -(1 row) - -SELECT id, provider_name FROM pg_tde_list_all_key_providers(); - id | provider_name -----+--------------- - 1 | file-provider -(1 row) - -SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); - id | provider_name -----+----------------- - -1 | reg_file-global - -2 | file-keyring - -3 | file-keyring2 -(3 rows) - -SELECT pg_tde_set_global_principal_key('test-db-principal-key', 'file-keyring', false); - pg_tde_set_global_principal_key ---------------------------------- - -(1 row) - --- fails -SELECT pg_tde_delete_global_key_provider('file-keyring'); - pg_tde_delete_global_key_provider ------------------------------------ - -(1 row) - -SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); - id | provider_name -----+----------------- - -1 | reg_file-global - -3 | file-keyring2 -(2 rows) - --- works -SELECT pg_tde_delete_global_key_provider('file-keyring2'); - pg_tde_delete_global_key_provider ------------------------------------ - -(1 row) - -SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); - id | provider_name -----+----------------- - -1 | reg_file-global -(1 row) - -DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/expected/keyprovider_dependency.out b/contrib/pg_tde/expected/keyprovider_dependency.out deleted file mode 100644 index e9133e52f72d5..0000000000000 --- a/contrib/pg_tde/expected/keyprovider_dependency.out +++ /dev/null @@ -1,34 +0,0 @@ -CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('mk-file','/tmp/pg_tde_test_keyring.per'); - pg_tde_add_key_provider_file ------------------------------- - 1 -(1 row) - -SELECT pg_tde_add_key_provider_file('free-file','/tmp/pg_tde_test_keyring_2.per'); - pg_tde_add_key_provider_file ------------------------------- - 2 -(1 row) - -SELECT pg_tde_add_key_provider_vault_v2('V2-vault','vault-token','percona.com/vault-v2/percona','/mount/dev','ca-cert-auth'); - pg_tde_add_key_provider_vault_v2 ----------------------------------- - 3 -(1 row) - -SELECT * FROM pg_tde_list_all_key_providers(); - id | provider_name | provider_type | options -----+---------------+---------------+----------------------------------------------------------------------------------------------------------------------------------------------- - 1 | mk-file | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring.per"} - 2 | free-file | file | {"type" : "file", "path" : "/tmp/pg_tde_test_keyring_2.per"} - 3 | V2-vault | vault-v2 | {"type" : "vault-v2", "url" : "percona.com/vault-v2/percona", "token" : "vault-token", "mountPath" : "/mount/dev", "caPath" : "ca-cert-auth"} -(3 rows) - -SELECT pg_tde_set_principal_key('test-db-principal-key','mk-file'); - pg_tde_set_principal_key --------------------------- - -(1 row) - -DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/expected/kmip_test.out b/contrib/pg_tde/expected/kmip_test.out index 71f5ac083d84c..3d905bbaeedb3 100644 --- a/contrib/pg_tde/expected/kmip_test.out +++ b/contrib/pg_tde/expected/kmip_test.out @@ -1,13 +1,13 @@ CREATE EXTENSION pg_tde; -SELECT pg_tde_add_key_provider_kmip('kmip-prov','127.0.0.1', 5696, '/tmp/server_certificate.pem', '/tmp/client_key_jane_doe.pem'); - pg_tde_add_key_provider_kmip ------------------------------- - 1 +SELECT pg_tde_add_database_key_provider_kmip('kmip-prov','127.0.0.1', 5696, '/tmp/server_certificate.pem', '/tmp/client_key_jane_doe.pem'); + pg_tde_add_database_key_provider_kmip +--------------------------------------- + 1 (1 row) -SELECT pg_tde_set_principal_key('kmip-principal-key','kmip-prov'); - pg_tde_set_principal_key --------------------------- +SELECT pg_tde_set_key_using_database_key_provider('kmip-key','kmip-prov'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- (1 row) @@ -27,5 +27,14 @@ SELECT * from test_enc; 3 | 3 (3 rows) +SELECT pg_tde_verify_key(); + pg_tde_verify_key +------------------- + +(1 row) + DROP TABLE test_enc; +-- Creating provider fails if we can't connect to kmip server +SELECT pg_tde_add_database_key_provider_kmip('will-not-work','127.0.0.1', 61, '/tmp/server_certificate.pem', '/tmp/client_key_jane_doe.pem'); +ERROR: SSL error: BIO_do_connect failed DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/expected/no_provider_error.out b/contrib/pg_tde/expected/no_provider_error.out deleted file mode 100644 index e4eb1693d21e3..0000000000000 --- a/contrib/pg_tde/expected/no_provider_error.out +++ /dev/null @@ -1,8 +0,0 @@ -CREATE EXTENSION pg_tde; --- should fail -CREATE TABLE t1 (n INT) USING tde_heap; -ERROR: failed to retrieve principal key. Create one using pg_tde_set_principal_key before using encrypted tables. --- should work -CREATE TABLE t2 (n INT) USING heap; -DROP TABLE t2; -DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/expected/partition_table.out b/contrib/pg_tde/expected/partition_table.out new file mode 100644 index 0000000000000..05cdc5708e66c --- /dev/null +++ b/contrib/pg_tde/expected/partition_table.out @@ -0,0 +1,90 @@ +CREATE EXTENSION pg_tde; +SELECT pg_tde_add_database_key_provider_file('database_keyring_provider','/tmp/pg_tde_keyring.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 +(1 row) + +SELECT pg_tde_set_key_using_database_key_provider('table_key','database_keyring_provider'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- + +(1 row) + +CREATE TABLE IF NOT EXISTS partitioned_table ( + id SERIAL, + data TEXT, + created_at DATE NOT NULL, + PRIMARY KEY (id, created_at) + ) PARTITION BY RANGE (created_at) USING tde_heap; +CREATE TABLE partition_q1_2024 PARTITION OF partitioned_table FOR VALUES FROM ('2024-01-01') TO ('2024-04-01') USING tde_heap; +CREATE TABLE partition_q2_2024 PARTITION OF partitioned_table FOR VALUES FROM ('2024-04-01') TO ('2024-07-01') USING heap; +CREATE TABLE partition_q3_2024 PARTITION OF partitioned_table FOR VALUES FROM ('2024-07-01') TO ('2024-10-01') USING tde_heap; +CREATE TABLE partition_q4_2024 PARTITION OF partitioned_table FOR VALUES FROM ('2024-10-01') TO ('2025-01-01') USING heap; +SELECT pg_tde_is_encrypted('partitioned_table'); + pg_tde_is_encrypted +--------------------- + +(1 row) + +SELECT pg_tde_is_encrypted('partition_q1_2024'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +SELECT pg_tde_is_encrypted('partition_q2_2024'); + pg_tde_is_encrypted +--------------------- + f +(1 row) + +SELECT pg_tde_is_encrypted('partition_q3_2024'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +SELECT pg_tde_is_encrypted('partition_q4_2024'); + pg_tde_is_encrypted +--------------------- + f +(1 row) + +ALTER TABLE partitioned_table SET ACCESS METHOD heap; +ALTER TABLE partition_q1_2024 SET ACCESS METHOD heap; +ALTER TABLE partition_q2_2024 SET ACCESS METHOD tde_heap; +ALTER TABLE partition_q3_2024 SET ACCESS METHOD heap; +ALTER TABLE partition_q4_2024 SET ACCESS METHOD tde_heap; +SELECT pg_tde_is_encrypted('partitioned_table'); + pg_tde_is_encrypted +--------------------- + +(1 row) + +SELECT pg_tde_is_encrypted('partition_q1_2024'); + pg_tde_is_encrypted +--------------------- + f +(1 row) + +SELECT pg_tde_is_encrypted('partition_q2_2024'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +SELECT pg_tde_is_encrypted('partition_q3_2024'); + pg_tde_is_encrypted +--------------------- + f +(1 row) + +SELECT pg_tde_is_encrypted('partition_q4_2024'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +DROP TABLE partitioned_table; +DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/expected/pg_tde_is_encrypted.out b/contrib/pg_tde/expected/pg_tde_is_encrypted.out index f9607ae10a0ae..b3f3812ac98c0 100644 --- a/contrib/pg_tde/expected/pg_tde_is_encrypted.out +++ b/contrib/pg_tde/expected/pg_tde_is_encrypted.out @@ -1,16 +1,16 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT * FROM pg_tde_principal_key_info(); +SELECT * FROM pg_tde_key_info(); ERROR: Principal key does not exists for the database -HINT: Use set_principal_key interface to set the principal key -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); - pg_tde_add_key_provider_file ------------------------------- - 1 +HINT: Use set_key interface to set the principal key +SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 (1 row) -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); - pg_tde_set_principal_key --------------------------- +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- (1 row) @@ -67,11 +67,11 @@ SELECT pg_tde_is_encrypted(NULL); (1 row) -SELECT key_provider_id, key_provider_name, principal_key_name - FROM pg_tde_principal_key_info(); - key_provider_id | key_provider_name | principal_key_name ------------------+-------------------+----------------------- - 1 | file-vault | test-db-principal-key +SELECT key_provider_id, key_provider_name, key_name + FROM pg_tde_key_info(); + key_provider_id | key_provider_name | key_name +-----------------+-------------------+------------- + 1 | file-vault | test-db-key (1 row) DROP TABLE test_part; diff --git a/contrib/pg_tde/expected/recreate_storage.out b/contrib/pg_tde/expected/recreate_storage.out index 0f8ce3d66c99d..58c3b3a055752 100644 --- a/contrib/pg_tde/expected/recreate_storage.out +++ b/contrib/pg_tde/expected/recreate_storage.out @@ -1,13 +1,13 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); - pg_tde_add_key_provider_file ------------------------------- - 1 +SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 (1 row) -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); - pg_tde_set_principal_key --------------------------- +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- (1 row) diff --git a/contrib/pg_tde/expected/relocate.out b/contrib/pg_tde/expected/relocate.out index fef8c84030243..af00e872ce9db 100644 --- a/contrib/pg_tde/expected/relocate.out +++ b/contrib/pg_tde/expected/relocate.out @@ -3,10 +3,10 @@ SET client_min_messages = 'warning'; DROP EXTENSION IF EXISTS pg_tde; CREATE SCHEMA other; CREATE EXTENSION pg_tde SCHEMA other; -SELECT other.pg_tde_add_key_provider_file('file-vault', '/tmp/pg_tde_test_keyring.per'); - pg_tde_add_key_provider_file ------------------------------- - 1 +SELECT other.pg_tde_add_database_key_provider_file('file-vault', '/tmp/pg_tde_test_keyring.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 (1 row) SELECT other.pg_tde_grant_key_viewer_to_role('public'); diff --git a/contrib/pg_tde/expected/subtransaction.out b/contrib/pg_tde/expected/subtransaction.out deleted file mode 100644 index 7508be79bcb17..0000000000000 --- a/contrib/pg_tde/expected/subtransaction.out +++ /dev/null @@ -1,30 +0,0 @@ -CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); - pg_tde_add_key_provider_file ------------------------------- - 1 -(1 row) - -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); - pg_tde_set_principal_key --------------------------- - -(1 row) - -BEGIN; -- Nesting level 1 -SAVEPOINT sp; -CREATE TABLE foo(s TEXT); -- Nesting level 2 -RELEASE SAVEPOINT sp; -SAVEPOINT sp; -CREATE TABLE bar(s TEXT); -- Nesting level 2 -ROLLBACK TO sp; -- Rollback should not affect first subtransaction -COMMIT; -BEGIN; -- Nesting level 1 -SAVEPOINT sp; -DROP TABLE foo; -- Nesting level 2 -RELEASE SAVEPOINT sp; -SAVEPOINT sp; -CREATE TABLE bar(s TEXT); -- Nesting level 2 -ROLLBACK TO sp; -- Rollback should not affect first subtransaction -COMMIT; -DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/expected/tablespace.out b/contrib/pg_tde/expected/tablespace.out index 6384afeaa4c66..4d7bffce68de6 100644 --- a/contrib/pg_tde/expected/tablespace.out +++ b/contrib/pg_tde/expected/tablespace.out @@ -1,13 +1,13 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); - pg_tde_add_key_provider_file ------------------------------- - 1 +SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 (1 row) -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); - pg_tde_set_principal_key --------------------------- +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- (1 row) diff --git a/contrib/pg_tde/expected/toast_decrypt.out b/contrib/pg_tde/expected/toast_decrypt.out index fff4e7744add7..4ca99eea46751 100644 --- a/contrib/pg_tde/expected/toast_decrypt.out +++ b/contrib/pg_tde/expected/toast_decrypt.out @@ -1,13 +1,13 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); - pg_tde_add_key_provider_file ------------------------------- - 1 +SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 (1 row) -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); - pg_tde_set_principal_key --------------------------- +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- (1 row) diff --git a/contrib/pg_tde/expected/toast_decrypt_1.out b/contrib/pg_tde/expected/toast_decrypt_1.out deleted file mode 100644 index 6765617555537..0000000000000 --- a/contrib/pg_tde/expected/toast_decrypt_1.out +++ /dev/null @@ -1,24 +0,0 @@ -CREATE EXTENSION IF NOT EXISTS pg_tde; -NOTICE: extension "pg_tde" already exists, skipping -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); - pg_tde_add_key_provider_file ------------------------------- - 2 -(1 row) - -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); - pg_tde_set_principal_key --------------------------- - -(1 row) - -CREATE TABLE src (f1 TEXT STORAGE EXTERNAL) USING tde_heap; -INSERT INTO src VALUES(repeat('abcdeF',1000)); -SELECT * FROM src; - f1 --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - abcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeFabcdeF -(1 row) - -DROP TABLE src; -DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/expected/vault_v2_test.out b/contrib/pg_tde/expected/vault_v2_test.out index 0629d847848bf..b4ff2a1ffdfac 100644 --- a/contrib/pg_tde/expected/vault_v2_test.out +++ b/contrib/pg_tde/expected/vault_v2_test.out @@ -1,13 +1,13 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; \getenv root_token ROOT_TOKEN -SELECT pg_tde_add_key_provider_vault_v2('vault-incorrect',:'root_token','http://127.0.0.1:8200','DUMMY-TOKEN',NULL); - pg_tde_add_key_provider_vault_v2 ----------------------------------- - 1 +SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect',:'root_token','http://127.0.0.1:8200','DUMMY-TOKEN',NULL); + pg_tde_add_database_key_provider_vault_v2 +------------------------------------------- + 1 (1 row) -- FAILS -SELECT pg_tde_set_principal_key('vault-v2-principal-key','vault-incorrect'); +SELECT pg_tde_set_key_using_database_key_provider('vault-v2-key','vault-incorrect'); ERROR: Invalid HTTP response from keyring provider "vault-incorrect": 404 CREATE TABLE test_enc( id SERIAL, @@ -15,16 +15,16 @@ CREATE TABLE test_enc( PRIMARY KEY (id) ) USING tde_heap; ERROR: principal key not configured -HINT: create one using pg_tde_set_principal_key before using encrypted tables -SELECT pg_tde_add_key_provider_vault_v2('vault-v2',:'root_token','http://127.0.0.1:8200','secret',NULL); - pg_tde_add_key_provider_vault_v2 ----------------------------------- - 2 +HINT: create one using pg_tde_set_key before using encrypted tables +SELECT pg_tde_add_database_key_provider_vault_v2('vault-v2',:'root_token','http://127.0.0.1:8200','secret',NULL); + pg_tde_add_database_key_provider_vault_v2 +------------------------------------------- + 2 (1 row) -SELECT pg_tde_set_principal_key('vault-v2-principal-key','vault-v2'); - pg_tde_set_principal_key --------------------------- +SELECT pg_tde_set_key_using_database_key_provider('vault-v2-key','vault-v2'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- (1 row) @@ -44,5 +44,14 @@ SELECT * from test_enc; 3 | 3 (3 rows) +SELECT pg_tde_verify_key(); + pg_tde_verify_key +------------------- + +(1 row) + DROP TABLE test_enc; +-- Creating provider fails if we can't connect to vault +SELECT pg_tde_add_database_key_provider_vault_v2('will-not-work', :'root_token', 'http://127.0.0.1:61', 'secret', NULL); +ERROR: HTTP(S) request to keyring provider "will-not-work" failed DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/meson.build b/contrib/pg_tde/meson.build index 70c8cbc52637c..59bb5a21c702f 100644 --- a/contrib/pg_tde/meson.build +++ b/contrib/pg_tde/meson.build @@ -1,46 +1,55 @@ - curldep = dependency('libcurl') pg_tde_sources = files( - 'src/pg_tde.c', - 'src/transam/pg_tde_xact_handler.c', - 'src/access/pg_tde_tdemap.c', - 'src/access/pg_tde_xlog.c', - 'src/access/pg_tde_xlog_encrypt.c', - - 'src/encryption/enc_tde.c', - 'src/encryption/enc_aes.c', - - 'src/keyring/keyring_curl.c', - 'src/keyring/keyring_file.c', - 'src/keyring/keyring_vault.c', - 'src/keyring/keyring_kmip.c', - 'src/keyring/keyring_kmip_impl.c', - 'src/keyring/keyring_api.c', - - 'src/smgr/pg_tde_smgr.c', - - 'src/catalog/tde_keyring.c', - 'src/catalog/tde_keyring_parse_opts.c', - 'src/catalog/tde_principal_key.c', - 'src/common/pg_tde_shmem.c', - 'src/common/pg_tde_utils.c', - 'src/pg_tde_defs.c', - 'src/pg_tde_event_capture.c', - 'src/pg_tde_guc.c', - 'src/pg_tde.c', + 'src/access/pg_tde_tdemap.c', + 'src/access/pg_tde_xlog.c', + 'src/access/pg_tde_xlog_smgr.c', + 'src/catalog/tde_keyring.c', + 'src/catalog/tde_keyring_parse_opts.c', + 'src/catalog/tde_principal_key.c', + 'src/common/pg_tde_shmem.c', + 'src/common/pg_tde_utils.c', + 'src/encryption/enc_aes.c', + 'src/encryption/enc_tde.c', + 'src/keyring/keyring_api.c', + 'src/keyring/keyring_curl.c', + 'src/keyring/keyring_file.c', + 'src/keyring/keyring_kmip.c', + 'src/keyring/keyring_kmip_impl.c', + 'src/keyring/keyring_vault.c', + 'src/pg_tde.c', + 'src/pg_tde_event_capture.c', + 'src/pg_tde_guc.c', + 'src/smgr/pg_tde_smgr.c', +) + +tde_frontend_sources = files( + 'src/access/pg_tde_tdemap.c', + 'src/access/pg_tde_xlog_smgr.c', + 'src/catalog/tde_keyring.c', + 'src/catalog/tde_keyring_parse_opts.c', + 'src/catalog/tde_principal_key.c', + 'src/common/pg_tde_utils.c', + 'src/encryption/enc_aes.c', + 'src/encryption/enc_tde.c', + 'src/keyring/keyring_api.c', + 'src/keyring/keyring_curl.c', + 'src/keyring/keyring_file.c', + 'src/keyring/keyring_kmip.c', + 'src/keyring/keyring_kmip_impl.c', + 'src/keyring/keyring_vault.c', ) incdir = include_directories('src/include', '.', 'src/libkmip/libkmip/include/') kmip = static_library( 'kmip', - files( - 'src/libkmip/libkmip/src/kmip.c', - 'src/libkmip/libkmip/src/kmip_bio.c', - 'src/libkmip/libkmip/src/kmip_locate.c', - 'src/libkmip/libkmip/src/kmip_memset.c' - ), + files( + 'src/libkmip/libkmip/src/kmip.c', + 'src/libkmip/libkmip/src/kmip_bio.c', + 'src/libkmip/libkmip/src/kmip_locate.c', + 'src/libkmip/libkmip/src/kmip_memset.c' + ), c_args: [ '-w' ], # This is a 3rd party, disable warnings completely include_directories: incdir ) @@ -71,38 +80,42 @@ install_data( kwargs: contrib_data_args, ) -# toast_descrypt needs to be the first test when running with pg_tde -# preinstalled and default_principal_key needs to run after key_provider. +# default_principal_key needs to run after key_provider. sql_tests = [ - 'toast_decrypt', - 'access_control', - 'alter_index', - 'cache_alloc', - 'change_access_method', - 'insert_update_delete', - 'key_provider', - 'keyprovider_dependency', - 'kmip_test', - 'pg_tde_is_encrypted', - 'relocate', - 'recreate_storage', - 'subtransaction', - 'tablespace', - 'vault_v2_test', - 'default_principal_key', + 'access_control', + 'alter_index', + 'cache_alloc', + 'change_access_method', + 'insert_update_delete', + 'key_provider', + 'kmip_test', + 'partition_table', + 'pg_tde_is_encrypted', + 'relocate', + 'recreate_storage', + 'tablespace', + 'toast_decrypt', + 'vault_v2_test', + 'version', + 'default_principal_key', ] tap_tests = [ - 't/001_basic.pl', - 't/002_rotate_key.pl', - 't/003_remote_config.pl', - 't/004_file_config.pl', - 't/005_multiple_extensions.pl', - 't/006_remote_vault_config.pl', - 't/007_tde_heap.pl', - 't/008_key_rotate_tablespace.pl', - 't/009_wal_encrypt.pl', - 't/010_change_key_provider.pl', + 't/001_basic.pl', + 't/002_rotate_key.pl', + 't/003_remote_config.pl', + 't/004_file_config.pl', + 't/005_multiple_extensions.pl', + 't/006_remote_vault_config.pl', + 't/007_tde_heap.pl', + 't/008_key_rotate_tablespace.pl', + 't/009_wal_encrypt.pl', + 't/010_change_key_provider.pl', + 't/011_unlogged_tables.pl', + 't/012_replication.pl', + 't/013_crash_recovery.pl', + 't/014_pg_waldump_basic.pl', + 't/015_pg_waldump_fullpage.pl', ] tests += { @@ -115,31 +128,14 @@ tests += { 'runningcheck': false, }, 'tap': { - 'tests': tap_tests }, + 'tests': tap_tests + }, } -# TODO: do not duplicate -tde_decrypt_sources = files( - 'src/access/pg_tde_tdemap.c', - 'src/access/pg_tde_xlog_encrypt.c', - 'src/catalog/tde_keyring.c', - 'src/catalog/tde_keyring_parse_opts.c', - 'src/catalog/tde_principal_key.c', - 'src/common/pg_tde_utils.c', - 'src/encryption/enc_aes.c', - 'src/encryption/enc_tde.c', - 'src/keyring/keyring_api.c', - 'src/keyring/keyring_curl.c', - 'src/keyring/keyring_file.c', - 'src/keyring/keyring_vault.c', - 'src/keyring/keyring_kmip.c', - 'src/keyring/keyring_kmip_impl.c', - ) - pg_tde_inc = incdir pg_tde_frontend = static_library('pg_tde_frontend', - tde_decrypt_sources, + tde_frontend_sources, c_pch: pch_postgres_h, c_args: ['-DFRONTEND'], kwargs: mod_args, diff --git a/contrib/pg_tde/pg_tde--1.0-rc.sql b/contrib/pg_tde/pg_tde--1.0-rc.sql index 9cd513a0f2f2e..dcd73b0ad675e 100644 --- a/contrib/pg_tde/pg_tde--1.0-rc.sql +++ b/contrib/pg_tde/pg_tde--1.0-rc.sql @@ -4,32 +4,32 @@ \echo Use "CREATE EXTENSION pg_tde" to load this file. \quit -- Key Provider Management -CREATE FUNCTION pg_tde_add_key_provider(provider_type TEXT, provider_name TEXT, options JSON) +CREATE FUNCTION pg_tde_add_database_key_provider(provider_type TEXT, provider_name TEXT, options JSON) RETURNS INT LANGUAGE C AS 'MODULE_PATHNAME'; -CREATE FUNCTION pg_tde_add_key_provider_file(provider_name TEXT, file_path TEXT) +CREATE FUNCTION pg_tde_add_database_key_provider_file(provider_name TEXT, file_path TEXT) RETURNS INT LANGUAGE SQL BEGIN ATOMIC -- JSON keys in the options must be matched to the keys in -- load_file_keyring_provider_options function. - SELECT pg_tde_add_key_provider('file', provider_name, + SELECT pg_tde_add_database_key_provider('file', provider_name, json_object('type' VALUE 'file', 'path' VALUE COALESCE(file_path, ''))); END; -CREATE FUNCTION pg_tde_add_key_provider_file(provider_name TEXT, file_path JSON) +CREATE FUNCTION pg_tde_add_database_key_provider_file(provider_name TEXT, file_path JSON) RETURNS INT LANGUAGE SQL BEGIN ATOMIC -- JSON keys in the options must be matched to the keys in -- load_file_keyring_provider_options function. - SELECT pg_tde_add_key_provider('file', provider_name, + SELECT pg_tde_add_database_key_provider('file', provider_name, json_object('type' VALUE 'file', 'path' VALUE file_path)); END; -CREATE FUNCTION pg_tde_add_key_provider_vault_v2(provider_name TEXT, +CREATE FUNCTION pg_tde_add_database_key_provider_vault_v2(provider_name TEXT, vault_token TEXT, vault_url TEXT, vault_mount_path TEXT, @@ -39,7 +39,7 @@ LANGUAGE SQL BEGIN ATOMIC -- JSON keys in the options must be matched to the keys in -- load_vaultV2_keyring_provider_options function. - SELECT pg_tde_add_key_provider('vault-v2', provider_name, + SELECT pg_tde_add_database_key_provider('vault-v2', provider_name, json_object('type' VALUE 'vault-v2', 'url' VALUE COALESCE(vault_url, ''), 'token' VALUE COALESCE(vault_token, ''), @@ -47,7 +47,7 @@ BEGIN ATOMIC 'caPath' VALUE COALESCE(vault_ca_path, ''))); END; -CREATE FUNCTION pg_tde_add_key_provider_vault_v2(provider_name TEXT, +CREATE FUNCTION pg_tde_add_database_key_provider_vault_v2(provider_name TEXT, vault_token JSON, vault_url JSON, vault_mount_path JSON, @@ -57,7 +57,7 @@ LANGUAGE SQL BEGIN ATOMIC -- JSON keys in the options must be matched to the keys in -- load_vaultV2_keyring_provider_options function. - SELECT pg_tde_add_key_provider('vault-v2', provider_name, + SELECT pg_tde_add_database_key_provider('vault-v2', provider_name, json_object('type' VALUE 'vault-v2', 'url' VALUE vault_url, 'token' VALUE vault_token, @@ -65,7 +65,7 @@ BEGIN ATOMIC 'caPath' VALUE vault_ca_path)); END; -CREATE FUNCTION pg_tde_add_key_provider_kmip(provider_name TEXT, +CREATE FUNCTION pg_tde_add_database_key_provider_kmip(provider_name TEXT, kmip_host TEXT, kmip_port INT, kmip_ca_path TEXT, @@ -75,7 +75,7 @@ LANGUAGE SQL BEGIN ATOMIC -- JSON keys in the options must be matched to the keys in -- load_kmip_keyring_provider_options function. - SELECT pg_tde_add_key_provider('kmip', provider_name, + SELECT pg_tde_add_database_key_provider('kmip', provider_name, json_object('type' VALUE 'kmip', 'host' VALUE COALESCE(kmip_host, ''), 'port' VALUE kmip_port, @@ -83,7 +83,7 @@ BEGIN ATOMIC 'certPath' VALUE COALESCE(kmip_cert_path, ''))); END; -CREATE FUNCTION pg_tde_add_key_provider_kmip(provider_name TEXT, +CREATE FUNCTION pg_tde_add_database_key_provider_kmip(provider_name TEXT, kmip_host JSON, kmip_port JSON, kmip_ca_path JSON, @@ -93,7 +93,7 @@ LANGUAGE SQL BEGIN ATOMIC -- JSON keys in the options must be matched to the keys in -- load_kmip_keyring_provider_options function. - SELECT pg_tde_add_key_provider('kmip', provider_name, + SELECT pg_tde_add_database_key_provider('kmip', provider_name, json_object('type' VALUE 'kmip', 'host' VALUE kmip_host, 'port' VALUE kmip_port, @@ -101,12 +101,8 @@ BEGIN ATOMIC 'certPath' VALUE kmip_cert_path)); END; -CREATE FUNCTION pg_tde_set_default_principal_key(principal_key_name TEXT, provider_name TEXT DEFAULT NULL, ensure_new_key BOOLEAN DEFAULT FALSE) -RETURNS VOID -AS 'MODULE_PATHNAME' -LANGUAGE C; -CREATE FUNCTION pg_tde_list_all_key_providers +CREATE FUNCTION pg_tde_list_all_database_key_providers (OUT id INT, OUT provider_name TEXT, OUT provider_type TEXT, @@ -223,32 +219,32 @@ BEGIN ATOMIC END; -- Key Provider Management -CREATE FUNCTION pg_tde_change_key_provider(provider_type TEXT, provider_name TEXT, options JSON) +CREATE FUNCTION pg_tde_change_database_key_provider(provider_type TEXT, provider_name TEXT, options JSON) RETURNS INT LANGUAGE C AS 'MODULE_PATHNAME'; -CREATE FUNCTION pg_tde_change_key_provider_file(provider_name TEXT, file_path TEXT) +CREATE FUNCTION pg_tde_change_database_key_provider_file(provider_name TEXT, file_path TEXT) RETURNS INT LANGUAGE SQL BEGIN ATOMIC -- JSON keys in the options must be matched to the keys in -- load_file_keyring_provider_options function. - SELECT pg_tde_change_key_provider('file', provider_name, + SELECT pg_tde_change_database_key_provider('file', provider_name, json_object('type' VALUE 'file', 'path' VALUE COALESCE(file_path, ''))); END; -CREATE FUNCTION pg_tde_change_key_provider_file(provider_name TEXT, file_path JSON) +CREATE FUNCTION pg_tde_change_database_key_provider_file(provider_name TEXT, file_path JSON) RETURNS INT LANGUAGE SQL BEGIN ATOMIC -- JSON keys in the options must be matched to the keys in -- load_file_keyring_provider_options function. - SELECT pg_tde_change_key_provider('file', provider_name, + SELECT pg_tde_change_database_key_provider('file', provider_name, json_object('type' VALUE 'file', 'path' VALUE file_path)); END; -CREATE FUNCTION pg_tde_change_key_provider_vault_v2(provider_name TEXT, +CREATE FUNCTION pg_tde_change_database_key_provider_vault_v2(provider_name TEXT, vault_token TEXT, vault_url TEXT, vault_mount_path TEXT, @@ -258,7 +254,7 @@ LANGUAGE SQL BEGIN ATOMIC -- JSON keys in the options must be matched to the keys in -- load_vaultV2_keyring_provider_options function. - SELECT pg_tde_change_key_provider('vault-v2', provider_name, + SELECT pg_tde_change_database_key_provider('vault-v2', provider_name, json_object('type' VALUE 'vault-v2', 'url' VALUE COALESCE(vault_url, ''), 'token' VALUE COALESCE(vault_token, ''), @@ -266,7 +262,7 @@ BEGIN ATOMIC 'caPath' VALUE COALESCE(vault_ca_path, ''))); END; -CREATE FUNCTION pg_tde_change_key_provider_vault_v2(provider_name TEXT, +CREATE FUNCTION pg_tde_change_database_key_provider_vault_v2(provider_name TEXT, vault_token JSON, vault_url JSON, vault_mount_path JSON, @@ -276,7 +272,7 @@ LANGUAGE SQL BEGIN ATOMIC -- JSON keys in the options must be matched to the keys in -- load_vaultV2_keyring_provider_options function. - SELECT pg_tde_change_key_provider('vault-v2', provider_name, + SELECT pg_tde_change_database_key_provider('vault-v2', provider_name, json_object('type' VALUE 'vault-v2', 'url' VALUE vault_url, 'token' VALUE vault_token, @@ -284,7 +280,7 @@ BEGIN ATOMIC 'caPath' VALUE vault_ca_path)); END; -CREATE FUNCTION pg_tde_change_key_provider_kmip(provider_name TEXT, +CREATE FUNCTION pg_tde_change_database_key_provider_kmip(provider_name TEXT, kmip_host TEXT, kmip_port INT, kmip_ca_path TEXT, @@ -294,7 +290,7 @@ LANGUAGE SQL BEGIN ATOMIC -- JSON keys in the options must be matched to the keys in -- load_kmip_keyring_provider_options function. - SELECT pg_tde_change_key_provider('kmip', provider_name, + SELECT pg_tde_change_database_key_provider('kmip', provider_name, json_object('type' VALUE 'kmip', 'host' VALUE COALESCE(kmip_host, ''), 'port' VALUE kmip_port, @@ -302,7 +298,7 @@ BEGIN ATOMIC 'certPath' VALUE COALESCE(kmip_cert_path, ''))); END; -CREATE FUNCTION pg_tde_change_key_provider_kmip(provider_name TEXT, +CREATE FUNCTION pg_tde_change_database_key_provider_kmip(provider_name TEXT, kmip_host JSON, kmip_port JSON, kmip_ca_path JSON, @@ -312,7 +308,7 @@ LANGUAGE SQL BEGIN ATOMIC -- JSON keys in the options must be matched to the keys in -- load_kmip_keyring_provider_options function. - SELECT pg_tde_change_key_provider('kmip', provider_name, + SELECT pg_tde_change_database_key_provider('kmip', provider_name, json_object('type' VALUE 'kmip', 'host' VALUE kmip_host, 'port' VALUE kmip_port, @@ -418,92 +414,73 @@ BEGIN ATOMIC 'certPath' VALUE kmip_cert_path)); END; - -CREATE FUNCTION pg_tde_internal_refresh_sequences(table_oid OID) -RETURNS VOID -AS -$BODY$ -DECLARE - rec RECORD; -BEGIN - FOR rec IN - SELECT s.relname AS sequence_name, - ns.nspname AS sequence_namespace, - se.seqstart AS sequence_start - FROM pg_class AS t - JOIN pg_attribute AS a - ON a.attrelid = t.oid - JOIN pg_depend AS d - ON d.refobjid = t.oid - AND d.refobjsubid = a.attnum - JOIN pg_class AS s - ON s.oid = d.objid - JOIN pg_sequence AS se - ON se.seqrelid = d.objid - JOIN pg_namespace AS ns - ON ns.oid = s.relnamespace - WHERE d.classid = 'pg_catalog.pg_class'::regclass - AND d.refclassid = 'pg_catalog.pg_class'::regclass - AND d.deptype IN ('i', 'a') - AND t.relkind IN ('r', 'P') - AND s.relkind = 'S' - AND t.oid = table_oid - LOOP - EXECUTE format('ALTER SEQUENCE %s.%s START %s', rec.sequence_namespace, rec.sequence_name, rec.sequence_start); - END LOOP; -END -$BODY$ -LANGUAGE plpgsql; - CREATE FUNCTION pg_tde_is_encrypted(relation regclass) RETURNS boolean STRICT LANGUAGE C AS 'MODULE_PATHNAME'; -CREATE FUNCTION pg_tde_set_principal_key(principal_key_name TEXT, provider_name TEXT DEFAULT NULL, ensure_new_key BOOLEAN DEFAULT FALSE) +CREATE FUNCTION pg_tde_set_key_using_database_key_provider(key_name TEXT, provider_name TEXT DEFAULT NULL, ensure_new_key BOOLEAN DEFAULT FALSE) RETURNS VOID LANGUAGE C AS 'MODULE_PATHNAME'; -CREATE FUNCTION pg_tde_set_global_principal_key(principal_key_name TEXT, provider_name TEXT DEFAULT NULL, ensure_new_key BOOLEAN DEFAULT FALSE) +CREATE FUNCTION pg_tde_set_key_using_global_key_provider(key_name TEXT, provider_name TEXT DEFAULT NULL, ensure_new_key BOOLEAN DEFAULT FALSE) RETURNS VOID LANGUAGE C AS 'MODULE_PATHNAME'; -CREATE FUNCTION pg_tde_set_server_principal_key(principal_key_name TEXT, provider_name TEXT DEFAULT NULL, ensure_new_key BOOLEAN DEFAULT FALSE) +CREATE FUNCTION pg_tde_set_server_key_using_global_key_provider(key_name TEXT, provider_name TEXT DEFAULT NULL, ensure_new_key BOOLEAN DEFAULT FALSE) RETURNS VOID LANGUAGE C AS 'MODULE_PATHNAME'; +CREATE FUNCTION pg_tde_set_default_key_using_global_key_provider(key_name TEXT, provider_name TEXT DEFAULT NULL, ensure_new_key BOOLEAN DEFAULT FALSE) +RETURNS VOID +AS 'MODULE_PATHNAME' +LANGUAGE C; + CREATE FUNCTION pg_tde_extension_initialize() RETURNS VOID LANGUAGE C AS 'MODULE_PATHNAME'; -CREATE FUNCTION pg_tde_verify_principal_key() +CREATE FUNCTION pg_tde_verify_key() RETURNS VOID LANGUAGE C AS 'MODULE_PATHNAME'; -CREATE FUNCTION pg_tde_verify_global_principal_key() +CREATE FUNCTION pg_tde_verify_server_key() RETURNS VOID LANGUAGE C AS 'MODULE_PATHNAME'; -CREATE FUNCTION pg_tde_principal_key_info() -RETURNS TABLE ( principal_key_name text, +CREATE FUNCTION pg_tde_verify_default_key() +RETURNS VOID +LANGUAGE C +AS 'MODULE_PATHNAME'; + +CREATE FUNCTION pg_tde_key_info() +RETURNS TABLE ( key_name text, + key_provider_name text, + key_provider_id integer, + key_creation_time timestamp with time zone) +LANGUAGE C +AS 'MODULE_PATHNAME'; + +CREATE FUNCTION pg_tde_server_key_info() +RETURNS TABLE ( key_name text, key_provider_name text, key_provider_id integer, - key_createion_time timestamp with time zone) + key_creation_time timestamp with time zone) LANGUAGE C AS 'MODULE_PATHNAME'; -CREATE FUNCTION pg_tde_global_principal_key_info() -RETURNS TABLE ( principal_key_name text, +CREATE FUNCTION pg_tde_default_key_info() +RETURNS TABLE ( key_name text, key_provider_name text, key_provider_id integer, - key_createion_time timestamp with time zone) + key_creation_time timestamp with time zone) LANGUAGE C AS 'MODULE_PATHNAME'; @@ -512,7 +489,7 @@ RETURNS VOID LANGUAGE C AS 'MODULE_PATHNAME'; -CREATE FUNCTION pg_tde_delete_key_provider(provider_name TEXT) +CREATE FUNCTION pg_tde_delete_database_key_provider(provider_name TEXT) RETURNS VOID LANGUAGE C AS 'MODULE_PATHNAME'; @@ -538,81 +515,47 @@ RETURNS event_trigger LANGUAGE C AS 'MODULE_PATHNAME'; -CREATE EVENT TRIGGER pg_tde_trigger_create_index +CREATE EVENT TRIGGER pg_tde_ddl_start ON ddl_command_start EXECUTE FUNCTION pg_tde_ddl_command_start_capture(); -ALTER EVENT TRIGGER pg_tde_trigger_create_index ENABLE ALWAYS; +ALTER EVENT TRIGGER pg_tde_ddl_start ENABLE ALWAYS; -CREATE EVENT TRIGGER pg_tde_trigger_create_index_2 +CREATE EVENT TRIGGER pg_tde_ddl_end ON ddl_command_end EXECUTE FUNCTION pg_tde_ddl_command_end_capture(); -ALTER EVENT TRIGGER pg_tde_trigger_create_index_2 ENABLE ALWAYS; +ALTER EVENT TRIGGER pg_tde_ddl_end ENABLE ALWAYS; -- Per database extension initialization SELECT pg_tde_extension_initialize(); -CREATE FUNCTION pg_tde_grant_global_key_management_to_role( - target_role TEXT) -RETURNS VOID -LANGUAGE plpgsql -SET search_path = @extschema@ -AS $$ -BEGIN - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_global_key_provider(text, text, JSON) TO %I', target_role); - - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_global_key_provider_file(text, json) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_global_key_provider_file(text, text) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_global_key_provider_vault_v2(text, text, text, text, text) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_global_key_provider_vault_v2(text, JSON, JSON, JSON, JSON) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_global_key_provider_kmip(text, text, int, text, text) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_global_key_provider_kmip(text, JSON, JSON, JSON, JSON) TO %I', target_role); - - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_global_key_provider(text, text, JSON) TO %I', target_role); - - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_global_key_provider_file(text, json) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_global_key_provider_file(text, text) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_global_key_provider_vault_v2(text, text, text, text, text) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_global_key_provider_vault_v2(text, JSON, JSON, JSON, JSON) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_global_key_provider_kmip(text, text, int, text, text) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_global_key_provider_kmip(text, JSON, JSON, JSON, JSON) TO %I', target_role); - - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_delete_global_key_provider(text) TO %I', target_role); - - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_set_global_principal_key(text, text, BOOLEAN) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_set_server_principal_key(text, text, BOOLEAN) TO %I', target_role); - - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_set_default_principal_key(text, text, BOOLEAN) TO %I', target_role); -END; -$$; - -CREATE FUNCTION pg_tde_grant_local_key_management_to_role( +CREATE FUNCTION pg_tde_grant_database_key_management_to_role( target_role TEXT) RETURNS VOID LANGUAGE plpgsql SET search_path = @extschema@ AS $$ BEGIN - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_key_provider(text, text, JSON) TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_database_key_provider(text, text, JSON) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_key_provider_file(text, json) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_key_provider_file(text, text) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_key_provider_vault_v2(text, text, text, text, text) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_key_provider_vault_v2(text, JSON, JSON, JSON, JSON) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_key_provider_kmip(text, text, int, text, text) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_key_provider_kmip(text, JSON, JSON, JSON, JSON) TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_database_key_provider_file(text, json) TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_database_key_provider_file(text, text) TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_database_key_provider_vault_v2(text, text, text, text, text) TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_database_key_provider_vault_v2(text, JSON, JSON, JSON, JSON) TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_database_key_provider_kmip(text, text, int, text, text) TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_add_database_key_provider_kmip(text, JSON, JSON, JSON, JSON) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_key_provider(text, text, JSON) TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_database_key_provider(text, text, JSON) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_key_provider_file(text, json) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_key_provider_file(text, text) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_key_provider_vault_v2(text, text, text,text,text) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_key_provider_vault_v2(text, JSON, JSON,JSON,JSON) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_key_provider_kmip(text, text, int, text, text) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_key_provider_kmip(text, JSON, JSON, JSON, JSON) TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_database_key_provider_file(text, json) TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_database_key_provider_file(text, text) TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_database_key_provider_vault_v2(text, text, text,text,text) TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_database_key_provider_vault_v2(text, JSON, JSON,JSON,JSON) TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_database_key_provider_kmip(text, text, int, text, text) TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_change_database_key_provider_kmip(text, JSON, JSON, JSON, JSON) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_delete_key_provider(text) TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_delete_database_key_provider(text) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_set_principal_key(text, text, BOOLEAN) TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_set_key_using_database_key_provider(text, text, BOOLEAN) TO %I', target_role); END; $$; @@ -623,78 +566,47 @@ LANGUAGE plpgsql SET search_path = @extschema@ AS $$ BEGIN - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_list_all_key_providers() TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_list_all_database_key_providers() TO %I', target_role); EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_list_all_global_key_providers() TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_principal_key_info() TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_global_principal_key_info() TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_verify_principal_key() TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_verify_global_principal_key() TO %I', target_role); -END; -$$; + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_key_info() TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_server_key_info() TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_default_key_info() TO %I', target_role); -CREATE FUNCTION pg_tde_revoke_global_key_management_from_role( - target_role TEXT) -RETURNS VOID -LANGUAGE plpgsql -SET search_path = @extschema@ -AS $$ -BEGIN - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_global_key_provider(text, text, JSON) FROM %I', target_role); - - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_global_key_provider_file(text, json) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_global_key_provider_file(text, text) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_global_key_provider_vault_v2(text, text, text, text, text) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_global_key_provider_vault_v2(text, JSON, JSON, JSON, JSON) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_global_key_provider_kmip(text, text, int, text, text) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_global_key_provider_kmip(text, JSON, JSON, JSON, JSON) FROM %I', target_role); - - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_global_key_provider(text, text, JSON) FROM %I', target_role); - - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_global_key_provider_file(text, json) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_global_key_provider_file(text, text) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_global_key_provider_vault_v2(text, text, text, text, text) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_global_key_provider_vault_v2(text, JSON, JSON, JSON, JSON) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_global_key_provider_kmip(text, text, int, text, text) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_global_key_provider_kmip(text, JSON, JSON, JSON, JSON) FROM %I', target_role); - - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_delete_global_key_provider(text) FROM %I', target_role); - - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_set_global_principal_key(text, text, BOOLEAN) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_set_server_principal_key(text, text, BOOLEAN) FROM %I', target_role); - - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_set_default_principal_key(text, text, BOOLEAN) FROM %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_verify_key() TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_verify_server_key() TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_verify_default_key() TO %I', target_role); END; $$; -CREATE FUNCTION pg_tde_revoke_local_key_management_from_role( +CREATE FUNCTION pg_tde_revoke_database_key_management_from_role( target_role TEXT) RETURNS VOID LANGUAGE plpgsql SET search_path = @extschema@ AS $$ BEGIN - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_key_provider(text, text, JSON) FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_database_key_provider(text, text, JSON) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_key_provider_file(text, json) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_key_provider_file(text, text) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_key_provider_vault_v2(text, text, text, text, text) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_key_provider_vault_v2(text, JSON, JSON, JSON, JSON) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_key_provider_kmip(text, text, int, text, text) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_key_provider_kmip(text, JSON, JSON, JSON, JSON) FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_database_key_provider_file(text, json) FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_database_key_provider_file(text, text) FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_database_key_provider_vault_v2(text, text, text, text, text) FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_database_key_provider_vault_v2(text, JSON, JSON, JSON, JSON) FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_database_key_provider_kmip(text, text, int, text, text) FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_add_database_key_provider_kmip(text, JSON, JSON, JSON, JSON) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_key_provider(text, text, JSON) FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_database_key_provider(text, text, JSON) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_key_provider_file(text, json) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_key_provider_file(text, text) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_key_provider_vault_v2(text, text, text, text, text) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_key_provider_vault_v2(text, JSON, JSON, JSON, JSON) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_key_provider_kmip(text, text, int, text, text) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_key_provider_kmip(text, JSON, JSON, JSON, JSON) FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_database_key_provider_file(text, json) FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_database_key_provider_file(text, text) FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_database_key_provider_vault_v2(text, text, text, text, text) FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_database_key_provider_vault_v2(text, JSON, JSON, JSON, JSON) FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_database_key_provider_kmip(text, text, int, text, text) FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_change_database_key_provider_kmip(text, JSON, JSON, JSON, JSON) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_delete_key_provider(text) FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_delete_database_key_provider(text) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_set_principal_key(text, text, BOOLEAN) FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_set_key_using_database_key_provider(text, text, BOOLEAN) FROM %I', target_role); END; $$; @@ -705,13 +617,16 @@ LANGUAGE plpgsql SET search_path = @extschema@ AS $$ BEGIN - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_list_all_key_providers() FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_list_all_database_key_providers() FROM %I', target_role); EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_list_all_global_key_providers() FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_principal_key_info() FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_global_principal_key_info() FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_verify_principal_key() FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_verify_global_principal_key() FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_key_info() FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_server_key_info() FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_default_key_info() FROM %I', target_role); + + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_verify_key() FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_verify_server_key() FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_verify_default_key() FROM %I', target_role); END; $$; @@ -722,13 +637,11 @@ LANGUAGE plpgsql SET search_path = @extschema@ AS $$ BEGIN - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_grant_global_key_management_to_role(TEXT) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_grant_local_key_management_to_role(TEXT) TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_grant_database_key_management_to_role(TEXT) TO %I', target_role); EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_grant_grant_management_to_role(TEXT) TO %I', target_role); EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_grant_key_viewer_to_role(TEXT) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_revoke_global_key_management_from_role(TEXT) TO %I', target_role); - EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_revoke_local_key_management_from_role(TEXT) TO %I', target_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_revoke_database_key_management_from_role(TEXT) TO %I', target_role); EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_revoke_grant_management_from_role(TEXT) TO %I', target_role); EXECUTE format('GRANT EXECUTE ON FUNCTION pg_tde_revoke_key_viewer_from_role(TEXT) TO %I', target_role); END; @@ -741,20 +654,17 @@ LANGUAGE plpgsql SET search_path = @extschema@ AS $$ BEGIN - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_grant_global_key_management_to_role(TEXT) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_grant_local_key_management_to_role(TEXT) FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_grant_database_key_management_to_role(TEXT) FROM %I', target_role); EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_grant_grant_management_to_role(TEXT) FROM %I', target_role); EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_grant_key_viewer_to_role(TEXT) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_revoke_global_key_management_from_role(TEXT) FROM %I', target_role); - EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_revoke_local_key_management_from_role(TEXT) FROM %I', target_role); + EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_revoke_database_key_management_from_role(TEXT) FROM %I', target_role); EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_revoke_grant_management_from_role(TEXT) FROM %I', target_role); EXECUTE format('REVOKE EXECUTE ON FUNCTION pg_tde_revoke_key_viewer_from_role(TEXT) FROM %I', target_role); END; $$; -- Revoking all the privileges from the public role -SELECT pg_tde_revoke_local_key_management_from_role('public'); -SELECT pg_tde_revoke_global_key_management_from_role('public'); +SELECT pg_tde_revoke_database_key_management_from_role('public'); SELECT pg_tde_revoke_grant_management_from_role('public'); SELECT pg_tde_revoke_key_viewer_from_role('public'); diff --git a/contrib/pg_tde/sql/access_control.sql b/contrib/pg_tde/sql/access_control.sql index fc6d76ac1f026..9ec1b36d733b0 100644 --- a/contrib/pg_tde/sql/access_control.sql +++ b/contrib/pg_tde/sql/access_control.sql @@ -5,22 +5,44 @@ CREATE USER regress_pg_tde_access_control; SET ROLE regress_pg_tde_access_control; -- should throw access denied -SELECT pg_tde_add_key_provider_file('file-vault', '/tmp/pg_tde_test_keyring.per'); -SELECT pg_tde_set_principal_key('test-db-principal-key', 'file-vault'); +SELECT pg_tde_add_database_key_provider_file('local-file-provider', '/tmp/pg_tde_test_keyring.per'); +SELECT pg_tde_set_key_using_database_key_provider('test-db-key', 'local-file-provider'); +SELECT pg_tde_add_global_key_provider_file('global-file-provider', '/tmp/pg_tde_test_keyring.per'); +SELECT pg_tde_set_key_using_global_key_provider('test-db-key', 'global-file-provider'); +SELECT pg_tde_set_server_key_using_global_key_provider('wal-key','global-file-provider'); +SELECT pg_tde_set_default_key_using_global_key_provider('def-key', 'global-file-provider'); +SELECT pg_tde_delete_database_key_provider('local-file-provider'); +SELECT pg_tde_delete_global_key_provider('global-file-provider'); +SELECT pg_tde_list_all_database_key_providers(); +SELECT pg_tde_list_all_global_key_providers(); +SELECT pg_tde_key_info(); +SELECT pg_tde_server_key_info(); +SELECT pg_tde_default_key_info(); +SELECT pg_tde_verify_key(); +SELECT pg_tde_verify_server_key(); +SELECT pg_tde_verify_default_key(); RESET ROLE; -SELECT pg_tde_grant_local_key_management_to_role('regress_pg_tde_access_control'); +SELECT pg_tde_grant_database_key_management_to_role('regress_pg_tde_access_control'); SELECT pg_tde_grant_key_viewer_to_role('regress_pg_tde_access_control'); SET ROLE regress_pg_tde_access_control; -- should now be allowed -SELECT pg_tde_add_key_provider_file('file-vault', '/tmp/pg_tde_test_keyring.per'); -SELECT pg_tde_add_key_provider_file('file-2', '/tmp/pg_tde_test_keyring_2.per'); -SELECT pg_tde_set_principal_key('test-db-principal-key', 'file-vault'); -SELECT * FROM pg_tde_list_all_key_providers(); -SELECT principal_key_name, key_provider_name, key_provider_id FROM pg_tde_principal_key_info(); +SELECT pg_tde_add_database_key_provider_file('local-file-provider', '/tmp/pg_tde_test_keyring.per'); +SELECT pg_tde_set_key_using_database_key_provider('test-db-key', 'local-file-provider'); +SELECT * FROM pg_tde_list_all_database_key_providers(); +SELECT key_name, key_provider_name, key_provider_id FROM pg_tde_key_info(); +SELECT pg_tde_verify_key(); + +-- only superuser +SELECT pg_tde_add_global_key_provider_file('global-file-provider', '/tmp/pg_tde_test_keyring.per'); +SELECT pg_tde_change_global_key_provider_file('global-file-provider', '/tmp/pg_tde_test_keyring.per'); +SELECT pg_tde_delete_global_key_provider('global-file-provider'); +SELECT pg_tde_set_key_using_global_key_provider('key1', 'global-file-provider'); +SELECT pg_tde_set_default_key_using_global_key_provider('key1', 'global-file-provider'); +SELECT pg_tde_set_server_key_using_global_key_provider('key1', 'global-file-provider'); RESET ROLE; @@ -29,8 +51,13 @@ SELECT pg_tde_revoke_key_viewer_from_role('regress_pg_tde_access_control'); SET ROLE regress_pg_tde_access_control; -- verify the view access is revoked -SELECT * FROM pg_tde_list_all_key_providers(); -SELECT principal_key_name, key_provider_name, key_provider_id FROM pg_tde_principal_key_info(); +SELECT * FROM pg_tde_list_all_database_key_providers(); +SELECT key_name, key_provider_name, key_provider_id FROM pg_tde_key_info(); +SELECT pg_tde_verify_key(); +SELECT pg_tde_server_key_info(); +SELECT pg_tde_default_key_info(); +SELECT pg_tde_verify_server_key(); +SELECT pg_tde_verify_default_key(); RESET ROLE; diff --git a/contrib/pg_tde/sql/alter_index.sql b/contrib/pg_tde/sql/alter_index.sql index 7f578d3fb3a50..9dac7bea58338 100644 --- a/contrib/pg_tde/sql/alter_index.sql +++ b/contrib/pg_tde/sql/alter_index.sql @@ -1,7 +1,7 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); +SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault'); SET default_table_access_method = "tde_heap"; diff --git a/contrib/pg_tde/sql/cache_alloc.sql b/contrib/pg_tde/sql/cache_alloc.sql index 9e89ba2efb180..745fdacfc18d8 100644 --- a/contrib/pg_tde/sql/cache_alloc.sql +++ b/contrib/pg_tde/sql/cache_alloc.sql @@ -2,8 +2,8 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); +SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault'); do $$ DECLARE idx integer; diff --git a/contrib/pg_tde/sql/change_access_method.sql b/contrib/pg_tde/sql/change_access_method.sql index 34a0955679730..dc01fbed8955a 100644 --- a/contrib/pg_tde/sql/change_access_method.sql +++ b/contrib/pg_tde/sql/change_access_method.sql @@ -1,13 +1,13 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); +SELECT pg_tde_add_database_key_provider_file('file-vault', '/tmp/pg_tde_test_keyring.per'); +SELECT pg_tde_set_key_using_database_key_provider('test-db-key', 'file-vault'); CREATE TABLE country_table ( - country_id serial primary key, - country_name varchar(32) unique not null, - continent varchar(32) not null -) using tde_heap; + country_id serial primary key, + country_name varchar(32) unique not null, + continent varchar(32) not null +) USING tde_heap; INSERT INTO country_table (country_name, continent) VALUES ('Japan', 'Asia'), @@ -15,19 +15,12 @@ INSERT INTO country_table (country_name, continent) ('USA', 'North America'); SELECT * FROM country_table; - SELECT pg_tde_is_encrypted('country_table'); - SELECT pg_tde_is_encrypted('country_table_country_id_seq'); - SELECT pg_tde_is_encrypted('country_table_pkey'); -- Try changing the encrypted table to an unencrypted table -ALTER TABLE country_table SET access method heap; - -SELECT pg_tde_is_encrypted('country_table_country_id_seq'); - -SELECT pg_tde_is_encrypted('country_table_pkey'); +ALTER TABLE country_table SET ACCESS METHOD heap; -- Insert some more data INSERT INTO country_table (country_name, continent) @@ -37,57 +30,68 @@ INSERT INTO country_table (country_name, continent) SELECT * FROM country_table; SELECT pg_tde_is_encrypted('country_table'); - SELECT pg_tde_is_encrypted('country_table_country_id_seq'); - SELECT pg_tde_is_encrypted('country_table_pkey'); -- Change it back to encrypted -ALTER TABLE country_table SET access method tde_heap; +ALTER TABLE country_table SET ACCESS METHOD tde_heap; INSERT INTO country_table (country_name, continent) VALUES ('China', 'Asia'), ('Brazil', 'South America'), ('Australia', 'Oceania'); + SELECT * FROM country_table; SELECT pg_tde_is_encrypted('country_table'); - SELECT pg_tde_is_encrypted('country_table_country_id_seq'); - SELECT pg_tde_is_encrypted('country_table_pkey'); +-- Test that we honor the default value +SET default_table_access_method = 'heap'; + +ALTER TABLE country_table SET ACCESS METHOD DEFAULT; + +SELECT pg_tde_is_encrypted('country_table'); + +SET default_table_access_method = 'tde_heap'; + +ALTER TABLE country_table SET ACCESS METHOD DEFAULT; + +SELECT pg_tde_is_encrypted('country_table'); + +RESET default_table_access_method; + ALTER TABLE country_table ADD y text; -SELECT pg_tde_is_encrypted(('pg_toast.pg_toast_' || 'country_table'::regclass::oid)::regclass); +SELECT pg_tde_is_encrypted('pg_toast.pg_toast_' || 'country_table'::regclass::oid); CREATE TABLE country_table2 ( - country_id serial primary key, - country_name text unique not null, - continent text not null + country_id serial primary key, + country_name text unique not null, + continent text not null ); -SET pg_tde.enforce_encryption = ON; +SET pg_tde.enforce_encryption = on; CREATE TABLE country_table3 ( - country_id serial primary key, - country_name text unique not null, - continent text not null + country_id serial primary key, + country_name text unique not null, + continent text not null ) USING heap; -ALTER TABLE country_table SET access method heap; - -ALTER TABLE country_table2 SET access method tde_heap; +ALTER TABLE country_table SET ACCESS METHOD heap; +ALTER TABLE country_table2 SET ACCESS METHOD tde_heap; CREATE TABLE country_table3 ( - country_id serial primary key, - country_name text unique not null, - continent text not null -) using tde_heap; + country_id serial primary key, + country_name text unique not null, + continent text not null +) USING tde_heap; DROP TABLE country_table; DROP TABLE country_table2; DROP TABLE country_table3; -SET pg_tde.enforce_encryption = OFF; +SET pg_tde.enforce_encryption = off; DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/sql/default_principal_key.sql b/contrib/pg_tde/sql/default_principal_key.sql index 1f60541e05231..9891b7cd6c5af 100644 --- a/contrib/pg_tde/sql/default_principal_key.sql +++ b/contrib/pg_tde/sql/default_principal_key.sql @@ -1,16 +1,28 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; +CREATE EXTENSION IF NOT EXISTS pg_buffercache; -SELECT pg_tde_add_global_key_provider_file('file-provider','/tmp/pg_tde_regression_default_principal_key.per'); +SELECT pg_tde_add_global_key_provider_file('file-provider','/tmp/pg_tde_regression_default_key.per'); -SELECT pg_tde_set_default_principal_key('default-principal-key', 'file-provider', false); +-- Should fail: no default principal key for the server yet +SELECT pg_tde_verify_default_key(); + +-- Should fail: no default principal key for the server yet +SELECT key_provider_id, key_provider_name, key_name + FROM pg_tde_default_key_info(); + +SELECT pg_tde_set_default_key_using_global_key_provider('default-key', 'file-provider', false); +SELECT pg_tde_verify_default_key(); + +SELECT key_provider_id, key_provider_name, key_name + FROM pg_tde_default_key_info(); -- fails SELECT pg_tde_delete_global_key_provider('file-provider'); SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); -- Should fail: no principal key for the database yet -SELECT key_provider_id, key_provider_name, principal_key_name - FROM pg_tde_principal_key_info(); +SELECT key_provider_id, key_provider_name, key_name + FROM pg_tde_key_info(); -- Should succeed: "localizes" the default principal key for the database CREATE TABLE test_enc( @@ -22,8 +34,8 @@ CREATE TABLE test_enc( INSERT INTO test_enc (k) VALUES (1), (2), (3); -- Should succeed: create table localized the principal key -SELECT key_provider_id, key_provider_name, principal_key_name - FROM pg_tde_principal_key_info(); +SELECT key_provider_id, key_provider_name, key_name + FROM pg_tde_key_info(); SELECT current_database() AS regress_database \gset @@ -33,9 +45,10 @@ CREATE DATABASE regress_pg_tde_other; \c regress_pg_tde_other CREATE EXTENSION pg_tde; +CREATE EXTENSION pg_buffercache; --- Should fail: no principal key for the database yet -SELECT key_provider_id, key_provider_name, principal_key_name +-- TODO +SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_principal_key_info(); -- Should succeed: "localizes" the default principal key for the database @@ -48,20 +61,26 @@ CREATE TABLE test_enc( INSERT INTO test_enc (k) VALUES (1), (2), (3); -- Should succeed: create table localized the principal key -SELECT key_provider_id, key_provider_name, principal_key_name - FROM pg_tde_principal_key_info(); +SELECT key_provider_id, key_provider_name, key_name + FROM pg_tde_key_info(); \c :regress_database -SELECT pg_tde_set_default_principal_key('new-default-principal-key', 'file-provider', false); +CHECKPOINT; -SELECT key_provider_id, key_provider_name, principal_key_name - FROM pg_tde_principal_key_info(); +SELECT pg_tde_set_default_key_using_global_key_provider('new-default-key', 'file-provider', false); + +SELECT key_provider_id, key_provider_name, key_name + FROM pg_tde_key_info(); \c regress_pg_tde_other -SELECT key_provider_id, key_provider_name, principal_key_name - FROM pg_tde_principal_key_info(); +SELECT key_provider_id, key_provider_name, key_name + FROM pg_tde_key_info(); + +SELECT pg_buffercache_evict(bufferid) FROM pg_buffercache WHERE relfilenode = (SELECT relfilenode FROM pg_class WHERE oid = 'test_enc'::regclass); + +SELECT * FROM test_enc; DROP TABLE test_enc; @@ -69,8 +88,13 @@ DROP EXTENSION pg_tde CASCADE; \c :regress_database +SELECT pg_buffercache_evict(bufferid) FROM pg_buffercache WHERE relfilenode = (SELECT relfilenode FROM pg_class WHERE oid = 'test_enc'::regclass); + +SELECT * FROM test_enc; + DROP TABLE test_enc; DROP EXTENSION pg_tde CASCADE; +DROP EXTENSION pg_buffercache; DROP DATABASE regress_pg_tde_other; diff --git a/contrib/pg_tde/sql/delete_key_provider.sql b/contrib/pg_tde/sql/delete_key_provider.sql deleted file mode 100644 index 431c97d6cc8fa..0000000000000 --- a/contrib/pg_tde/sql/delete_key_provider.sql +++ /dev/null @@ -1,20 +0,0 @@ -CREATE EXTENSION IF NOT EXISTS pg_tde; - -SELECT * FROM pg_tde_principal_key_info(); - -SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per'); -SELECT * FROM pg_tde_list_all_key_providers(); -SELECT pg_tde_delete_key_provider('file-provider'); -SELECT * FROM pg_tde_list_all_key_providers(); - -SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per'); -SELECT * FROM pg_tde_list_all_key_providers(); -SELECT pg_tde_delete_key_provider('file-provider'); -SELECT * FROM pg_tde_list_all_key_providers(); - -SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per'); -SELECT * FROM pg_tde_list_all_key_providers(); -SELECT pg_tde_delete_key_provider('file-provider'); -SELECT * FROM pg_tde_list_all_key_providers(); - -DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/sql/insert_update_delete.sql b/contrib/pg_tde/sql/insert_update_delete.sql index 3231a220a7e36..a586bf302f451 100644 --- a/contrib/pg_tde/sql/insert_update_delete.sql +++ b/contrib/pg_tde/sql/insert_update_delete.sql @@ -1,7 +1,7 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); +SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault'); CREATE TABLE albums ( id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, diff --git a/contrib/pg_tde/sql/key_provider.sql b/contrib/pg_tde/sql/key_provider.sql index 9732440bdb070..78fcc71de83a0 100644 --- a/contrib/pg_tde/sql/key_provider.sql +++ b/contrib/pg_tde/sql/key_provider.sql @@ -1,30 +1,30 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT * FROM pg_tde_principal_key_info(); +SELECT * FROM pg_tde_key_info(); -SELECT pg_tde_add_key_provider_file('incorrect-file-provider', json_object('foo' VALUE '/tmp/pg_tde_test_keyring.per')); -SELECT * FROM pg_tde_list_all_key_providers(); +SELECT pg_tde_add_database_key_provider_file('incorrect-file-provider', json_object('foo' VALUE '/tmp/pg_tde_test_keyring.per')); +SELECT * FROM pg_tde_list_all_database_key_providers(); -SELECT pg_tde_add_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per'); -SELECT * FROM pg_tde_list_all_key_providers(); +SELECT pg_tde_add_database_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per'); +SELECT * FROM pg_tde_list_all_database_key_providers(); -SELECT pg_tde_add_key_provider_file('file-provider2','/tmp/pg_tde_test_keyring2.per'); -SELECT * FROM pg_tde_list_all_key_providers(); +SELECT pg_tde_add_database_key_provider_file('file-provider2','/tmp/pg_tde_test_keyring2.per'); +SELECT * FROM pg_tde_list_all_database_key_providers(); -SELECT pg_tde_verify_principal_key(); +SELECT pg_tde_verify_key(); -SELECT pg_tde_set_principal_key('test-db-principal-key','file-provider'); -SELECT pg_tde_verify_principal_key(); +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-provider'); +SELECT pg_tde_verify_key(); -SELECT pg_tde_change_key_provider_file('not-existent-provider','/tmp/pg_tde_test_keyring.per'); -SELECT * FROM pg_tde_list_all_key_providers(); +SELECT pg_tde_change_database_key_provider_file('not-existent-provider','/tmp/pg_tde_test_keyring.per'); +SELECT * FROM pg_tde_list_all_database_key_providers(); -SELECT pg_tde_change_key_provider_file('file-provider','/tmp/pg_tde_test_keyring_other.per'); -SELECT * FROM pg_tde_list_all_key_providers(); -SELECT pg_tde_verify_principal_key(); +SELECT pg_tde_change_database_key_provider_file('file-provider','/tmp/pg_tde_test_keyring_other.per'); +SELECT * FROM pg_tde_list_all_database_key_providers(); +SELECT pg_tde_verify_key(); -SELECT pg_tde_change_key_provider_file('file-provider', json_object('foo' VALUE '/tmp/pg_tde_test_keyring.per')); -SELECT * FROM pg_tde_list_all_key_providers(); +SELECT pg_tde_change_database_key_provider_file('file-provider', json_object('foo' VALUE '/tmp/pg_tde_test_keyring.per')); +SELECT * FROM pg_tde_list_all_database_key_providers(); SELECT pg_tde_add_global_key_provider_file('file-keyring','/tmp/pg_tde_test_keyring.per'); @@ -32,19 +32,17 @@ SELECT pg_tde_add_global_key_provider_file('file-keyring2','/tmp/pg_tde_test_key SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); --- TODO: verify that we can also can change the type of it - -- fails -SELECT pg_tde_delete_key_provider('file-provider'); -SELECT id, provider_name FROM pg_tde_list_all_key_providers(); +SELECT pg_tde_delete_database_key_provider('file-provider'); +SELECT id, provider_name FROM pg_tde_list_all_database_key_providers(); -- works -SELECT pg_tde_delete_key_provider('file-provider2'); -SELECT id, provider_name FROM pg_tde_list_all_key_providers(); +SELECT pg_tde_delete_database_key_provider('file-provider2'); +SELECT id, provider_name FROM pg_tde_list_all_database_key_providers(); SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); -SELECT pg_tde_set_global_principal_key('test-db-principal-key', 'file-keyring', false); +SELECT pg_tde_set_key_using_global_key_provider('test-db-key', 'file-keyring', false); -- fails SELECT pg_tde_delete_global_key_provider('file-keyring'); @@ -54,5 +52,7 @@ SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); SELECT pg_tde_delete_global_key_provider('file-keyring2'); SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); -DROP EXTENSION pg_tde; +-- Creating a file key provider fails if we can't open or create the file +SELECT pg_tde_add_database_key_provider_file('will-not-work','/cant-create-file-in-root.per'); +DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/sql/keyprovider_dependency.sql b/contrib/pg_tde/sql/keyprovider_dependency.sql deleted file mode 100644 index 2c56d2d9e38e2..0000000000000 --- a/contrib/pg_tde/sql/keyprovider_dependency.sql +++ /dev/null @@ -1,11 +0,0 @@ -CREATE EXTENSION IF NOT EXISTS pg_tde; - -SELECT pg_tde_add_key_provider_file('mk-file','/tmp/pg_tde_test_keyring.per'); -SELECT pg_tde_add_key_provider_file('free-file','/tmp/pg_tde_test_keyring_2.per'); -SELECT pg_tde_add_key_provider_vault_v2('V2-vault','vault-token','percona.com/vault-v2/percona','/mount/dev','ca-cert-auth'); - -SELECT * FROM pg_tde_list_all_key_providers(); - -SELECT pg_tde_set_principal_key('test-db-principal-key','mk-file'); - -DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/sql/kmip_test.sql b/contrib/pg_tde/sql/kmip_test.sql index 85db4e9766e53..d47467e5fda16 100644 --- a/contrib/pg_tde/sql/kmip_test.sql +++ b/contrib/pg_tde/sql/kmip_test.sql @@ -1,7 +1,7 @@ CREATE EXTENSION pg_tde; -SELECT pg_tde_add_key_provider_kmip('kmip-prov','127.0.0.1', 5696, '/tmp/server_certificate.pem', '/tmp/client_key_jane_doe.pem'); -SELECT pg_tde_set_principal_key('kmip-principal-key','kmip-prov'); +SELECT pg_tde_add_database_key_provider_kmip('kmip-prov','127.0.0.1', 5696, '/tmp/server_certificate.pem', '/tmp/client_key_jane_doe.pem'); +SELECT pg_tde_set_key_using_database_key_provider('kmip-key','kmip-prov'); CREATE TABLE test_enc( id SERIAL, @@ -15,6 +15,11 @@ INSERT INTO test_enc (k) VALUES (3); SELECT * from test_enc; +SELECT pg_tde_verify_key(); + DROP TABLE test_enc; +-- Creating provider fails if we can't connect to kmip server +SELECT pg_tde_add_database_key_provider_kmip('will-not-work','127.0.0.1', 61, '/tmp/server_certificate.pem', '/tmp/client_key_jane_doe.pem'); + DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/sql/no_provider_error.sql b/contrib/pg_tde/sql/no_provider_error.sql deleted file mode 100644 index abe2e18cbcc5e..0000000000000 --- a/contrib/pg_tde/sql/no_provider_error.sql +++ /dev/null @@ -1,11 +0,0 @@ -CREATE EXTENSION pg_tde; - --- should fail -CREATE TABLE t1 (n INT) USING tde_heap; - --- should work -CREATE TABLE t2 (n INT) USING heap; - -DROP TABLE t2; - -DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/sql/partition_table.sql b/contrib/pg_tde/sql/partition_table.sql new file mode 100644 index 0000000000000..2651ef71d726f --- /dev/null +++ b/contrib/pg_tde/sql/partition_table.sql @@ -0,0 +1,35 @@ +CREATE EXTENSION pg_tde; +SELECT pg_tde_add_database_key_provider_file('database_keyring_provider','/tmp/pg_tde_keyring.per'); +SELECT pg_tde_set_key_using_database_key_provider('table_key','database_keyring_provider'); +CREATE TABLE IF NOT EXISTS partitioned_table ( + id SERIAL, + data TEXT, + created_at DATE NOT NULL, + PRIMARY KEY (id, created_at) + ) PARTITION BY RANGE (created_at) USING tde_heap; + +CREATE TABLE partition_q1_2024 PARTITION OF partitioned_table FOR VALUES FROM ('2024-01-01') TO ('2024-04-01') USING tde_heap; +CREATE TABLE partition_q2_2024 PARTITION OF partitioned_table FOR VALUES FROM ('2024-04-01') TO ('2024-07-01') USING heap; +CREATE TABLE partition_q3_2024 PARTITION OF partitioned_table FOR VALUES FROM ('2024-07-01') TO ('2024-10-01') USING tde_heap; +CREATE TABLE partition_q4_2024 PARTITION OF partitioned_table FOR VALUES FROM ('2024-10-01') TO ('2025-01-01') USING heap; + +SELECT pg_tde_is_encrypted('partitioned_table'); +SELECT pg_tde_is_encrypted('partition_q1_2024'); +SELECT pg_tde_is_encrypted('partition_q2_2024'); +SELECT pg_tde_is_encrypted('partition_q3_2024'); +SELECT pg_tde_is_encrypted('partition_q4_2024'); + +ALTER TABLE partitioned_table SET ACCESS METHOD heap; +ALTER TABLE partition_q1_2024 SET ACCESS METHOD heap; +ALTER TABLE partition_q2_2024 SET ACCESS METHOD tde_heap; +ALTER TABLE partition_q3_2024 SET ACCESS METHOD heap; +ALTER TABLE partition_q4_2024 SET ACCESS METHOD tde_heap; + +SELECT pg_tde_is_encrypted('partitioned_table'); +SELECT pg_tde_is_encrypted('partition_q1_2024'); +SELECT pg_tde_is_encrypted('partition_q2_2024'); +SELECT pg_tde_is_encrypted('partition_q3_2024'); +SELECT pg_tde_is_encrypted('partition_q4_2024'); + +DROP TABLE partitioned_table; +DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/sql/pg_tde_is_encrypted.sql b/contrib/pg_tde/sql/pg_tde_is_encrypted.sql index aa6d0c07ac17d..ceb0831c5c486 100644 --- a/contrib/pg_tde/sql/pg_tde_is_encrypted.sql +++ b/contrib/pg_tde/sql/pg_tde_is_encrypted.sql @@ -1,9 +1,9 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT * FROM pg_tde_principal_key_info(); +SELECT * FROM pg_tde_key_info(); -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); +SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault'); CREATE TABLE test_enc( id SERIAL, @@ -33,8 +33,8 @@ SELECT relname, pg_tde_is_encrypted(relname) FROM (VALUES ('test_enc_pkey'), ('t SELECT pg_tde_is_encrypted(NULL); -SELECT key_provider_id, key_provider_name, principal_key_name - FROM pg_tde_principal_key_info(); +SELECT key_provider_id, key_provider_name, key_name + FROM pg_tde_key_info(); DROP TABLE test_part; DROP TABLE test_norm; diff --git a/contrib/pg_tde/sql/recreate_storage.sql b/contrib/pg_tde/sql/recreate_storage.sql index 4389144e5de7e..3296a87152767 100644 --- a/contrib/pg_tde/sql/recreate_storage.sql +++ b/contrib/pg_tde/sql/recreate_storage.sql @@ -1,7 +1,7 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); +SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault'); SET default_table_access_method = "tde_heap"; diff --git a/contrib/pg_tde/sql/relocate.sql b/contrib/pg_tde/sql/relocate.sql index a18cb38095133..d9ce03b34f937 100644 --- a/contrib/pg_tde/sql/relocate.sql +++ b/contrib/pg_tde/sql/relocate.sql @@ -6,7 +6,7 @@ CREATE SCHEMA other; CREATE EXTENSION pg_tde SCHEMA other; -SELECT other.pg_tde_add_key_provider_file('file-vault', '/tmp/pg_tde_test_keyring.per'); +SELECT other.pg_tde_add_database_key_provider_file('file-vault', '/tmp/pg_tde_test_keyring.per'); SELECT other.pg_tde_grant_key_viewer_to_role('public'); diff --git a/contrib/pg_tde/sql/subtransaction.sql b/contrib/pg_tde/sql/subtransaction.sql deleted file mode 100644 index 681d505092a7f..0000000000000 --- a/contrib/pg_tde/sql/subtransaction.sql +++ /dev/null @@ -1,25 +0,0 @@ -CREATE EXTENSION IF NOT EXISTS pg_tde; - -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); - - -BEGIN; -- Nesting level 1 -SAVEPOINT sp; -CREATE TABLE foo(s TEXT); -- Nesting level 2 -RELEASE SAVEPOINT sp; -SAVEPOINT sp; -CREATE TABLE bar(s TEXT); -- Nesting level 2 -ROLLBACK TO sp; -- Rollback should not affect first subtransaction -COMMIT; - -BEGIN; -- Nesting level 1 -SAVEPOINT sp; -DROP TABLE foo; -- Nesting level 2 -RELEASE SAVEPOINT sp; -SAVEPOINT sp; -CREATE TABLE bar(s TEXT); -- Nesting level 2 -ROLLBACK TO sp; -- Rollback should not affect first subtransaction -COMMIT; - -DROP EXTENSION pg_tde; \ No newline at end of file diff --git a/contrib/pg_tde/sql/tablespace.sql b/contrib/pg_tde/sql/tablespace.sql index 102e8b755c535..7e2abce87ca13 100644 --- a/contrib/pg_tde/sql/tablespace.sql +++ b/contrib/pg_tde/sql/tablespace.sql @@ -1,7 +1,7 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); +SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault'); CREATE TABLE test(num1 bigint, num2 double precision, t text) USING tde_heap; INSERT INTO test(num1, num2, t) diff --git a/contrib/pg_tde/sql/toast_decrypt.sql b/contrib/pg_tde/sql/toast_decrypt.sql index 073e6bf27f83e..4cd6cf513f618 100644 --- a/contrib/pg_tde/sql/toast_decrypt.sql +++ b/contrib/pg_tde/sql/toast_decrypt.sql @@ -1,7 +1,7 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); +SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault'); CREATE TABLE src (f1 TEXT STORAGE EXTERNAL) USING tde_heap; INSERT INTO src VALUES(repeat('abcdeF',1000)); diff --git a/contrib/pg_tde/sql/vault_v2_test.sql b/contrib/pg_tde/sql/vault_v2_test.sql index 0e210dc1a65ba..205aa1118533f 100644 --- a/contrib/pg_tde/sql/vault_v2_test.sql +++ b/contrib/pg_tde/sql/vault_v2_test.sql @@ -2,9 +2,9 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; \getenv root_token ROOT_TOKEN -SELECT pg_tde_add_key_provider_vault_v2('vault-incorrect',:'root_token','http://127.0.0.1:8200','DUMMY-TOKEN',NULL); +SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect',:'root_token','http://127.0.0.1:8200','DUMMY-TOKEN',NULL); -- FAILS -SELECT pg_tde_set_principal_key('vault-v2-principal-key','vault-incorrect'); +SELECT pg_tde_set_key_using_database_key_provider('vault-v2-key','vault-incorrect'); CREATE TABLE test_enc( id SERIAL, @@ -12,8 +12,8 @@ CREATE TABLE test_enc( PRIMARY KEY (id) ) USING tde_heap; -SELECT pg_tde_add_key_provider_vault_v2('vault-v2',:'root_token','http://127.0.0.1:8200','secret',NULL); -SELECT pg_tde_set_principal_key('vault-v2-principal-key','vault-v2'); +SELECT pg_tde_add_database_key_provider_vault_v2('vault-v2',:'root_token','http://127.0.0.1:8200','secret',NULL); +SELECT pg_tde_set_key_using_database_key_provider('vault-v2-key','vault-v2'); CREATE TABLE test_enc( id SERIAL, @@ -27,6 +27,11 @@ INSERT INTO test_enc (k) VALUES (3); SELECT * from test_enc; +SELECT pg_tde_verify_key(); + DROP TABLE test_enc; +-- Creating provider fails if we can't connect to vault +SELECT pg_tde_add_database_key_provider_vault_v2('will-not-work', :'root_token', 'http://127.0.0.1:61', 'secret', NULL); + DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/src/access/pg_tde_tdemap.c b/contrib/pg_tde/src/access/pg_tde_tdemap.c index 5392b976097b2..91fa23a53b82a 100644 --- a/contrib/pg_tde/src/access/pg_tde_tdemap.c +++ b/contrib/pg_tde/src/access/pg_tde_tdemap.c @@ -13,7 +13,6 @@ #include "postgres.h" #include "access/pg_tde_tdemap.h" #include "common/file_perm.h" -#include "transam/pg_tde_xact_handler.h" #include "storage/fd.h" #include "utils/wait_event.h" #include "utils/memutils.h" @@ -66,7 +65,7 @@ typedef struct TDEFileHeader { int32 file_version; - TDEPrincipalKeyInfo principal_key_info; + TDESignedPrincipalKeyInfo signed_key_info; } TDEFileHeader; typedef struct RelKeyCacheRec @@ -110,77 +109,113 @@ static WALKeyCacheRec *tde_wal_key_cache = NULL; static WALKeyCacheRec *tde_wal_key_last_rec = NULL; static InternalKey *pg_tde_get_key_from_file(const RelFileLocator *rlocator, uint32 key_type); -static TDEMapEntry *pg_tde_find_map_entry(const RelFileLocator *rlocator, uint32 key_type, char *db_map_path); +static bool pg_tde_find_map_entry(const RelFileLocator *rlocator, uint32 key_type, char *db_map_path, TDEMapEntry *map_entry); static InternalKey *tde_decrypt_rel_key(TDEPrincipalKey *principal_key, TDEMapEntry *map_entry); static int pg_tde_open_file_basic(const char *tde_filename, int fileFlags, bool ignore_missing); static void pg_tde_file_header_read(const char *tde_filename, int fd, TDEFileHeader *fheader, off_t *bytes_read); -static bool pg_tde_read_one_map_entry(int fd, const RelFileLocator *rlocator, int flags, TDEMapEntry *map_entry, off_t *offset); -static TDEMapEntry *pg_tde_read_one_map_entry2(int keydata_fd, int32 key_index, TDEPrincipalKey *principal_key); +static bool pg_tde_read_one_map_entry(int fd, TDEMapEntry *map_entry, off_t *offset); +static void pg_tde_read_one_map_entry2(int keydata_fd, int32 key_index, TDEMapEntry *map_entry, Oid databaseId); static int pg_tde_open_file_read(const char *tde_filename, off_t *curr_pos); static InternalKey *pg_tde_get_key_from_cache(const RelFileLocator *rlocator, uint32 key_type); static WALKeyCacheRec *pg_tde_add_wal_key_to_cache(InternalKey *cached_key, XLogRecPtr start_lsn); static InternalKey *pg_tde_put_key_into_cache(const RelFileLocator *locator, InternalKey *key); #ifndef FRONTEND -static InternalKey *pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, uint32 entry_type); -static InternalKey *pg_tde_create_local_key(const RelFileLocator *newrlocator, uint32 entry_type); +static InternalKey *pg_tde_create_smgr_key_temp(const RelFileLocator *newrlocator); +static InternalKey *pg_tde_create_smgr_key_perm(const RelFileLocator *newrlocator); static void pg_tde_generate_internal_key(InternalKey *int_key, uint32 entry_type); -static int pg_tde_file_header_write(const char *tde_filename, int fd, TDEPrincipalKeyInfo *principal_key_info, off_t *bytes_written); -static off_t pg_tde_write_one_map_entry(int fd, const TDEMapEntry *map_entry, off_t *offset, const char *db_map_path); -static void pg_tde_write_key_map_entry(const RelFileLocator *rlocator, InternalKey *rel_key_data, TDEPrincipalKey *principal_key, bool write_xlog); -static bool pg_tde_delete_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_t offset); -static int keyrotation_init_file(TDEPrincipalKeyInfo *new_principal_key_info, char *rotated_filename, char *filename, off_t *curr_pos); +static int pg_tde_file_header_write(const char *tde_filename, int fd, const TDESignedPrincipalKeyInfo *signed_key_info, off_t *bytes_written); +static void pg_tde_sign_principal_key_info(TDESignedPrincipalKeyInfo *signed_key_info, const TDEPrincipalKey *principal_key); +static void pg_tde_write_one_map_entry(int fd, const TDEMapEntry *map_entry, off_t *offset, const char *db_map_path); +static void pg_tde_write_key_map_entry(const RelFileLocator *rlocator, InternalKey *rel_key_data, TDEPrincipalKey *principal_key); +static int keyrotation_init_file(const TDESignedPrincipalKeyInfo *signed_key_info, char *rotated_filename, const char *filename, off_t *curr_pos); static void finalize_key_rotation(const char *path_old, const char *path_new); -static int pg_tde_open_file_write(const char *tde_filename, TDEPrincipalKeyInfo *principal_key_info, bool truncate, off_t *curr_pos); +static int pg_tde_open_file_write(const char *tde_filename, const TDESignedPrincipalKeyInfo *signed_key_info, bool truncate, off_t *curr_pos); static void update_wal_keys_cache(void); InternalKey * pg_tde_create_smgr_key(const RelFileLocatorBackend *newrlocator) { if (RelFileLocatorBackendIsTemp(*newrlocator)) - return pg_tde_create_local_key(&newrlocator->locator, TDE_KEY_TYPE_SMGR); + return pg_tde_create_smgr_key_temp(&newrlocator->locator); else - return pg_tde_create_key_map_entry(&newrlocator->locator, TDE_KEY_TYPE_SMGR); + return pg_tde_create_smgr_key_perm(&newrlocator->locator); +} + +static InternalKey * +pg_tde_create_smgr_key_temp(const RelFileLocator *newrlocator) +{ + InternalKey int_key; + + pg_tde_generate_internal_key(&int_key, TDE_KEY_TYPE_SMGR); + + return pg_tde_put_key_into_cache(newrlocator, &int_key); } -/* - * Generate an encrypted key for the relation and store it in the keymap file. - */ static InternalKey * -pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, uint32 entry_type) +pg_tde_create_smgr_key_perm(const RelFileLocator *newrlocator) { InternalKey rel_key_data; TDEPrincipalKey *principal_key; LWLock *lock_pk = tde_lwlock_enc_keys(); + XLogRelKey xlrec = { + .rlocator = *newrlocator, + }; - pg_tde_generate_internal_key(&rel_key_data, entry_type); + pg_tde_generate_internal_key(&rel_key_data, TDE_KEY_TYPE_SMGR); LWLockAcquire(lock_pk, LW_EXCLUSIVE); principal_key = GetPrincipalKey(newrlocator->dbOid, LW_EXCLUSIVE); if (principal_key == NULL) { ereport(ERROR, - (errmsg("principal key not configured"), - errhint("create one using pg_tde_set_principal_key before using encrypted tables"))); + errmsg("principal key not configured"), + errhint("create one using pg_tde_set_key before using encrypted tables")); } + /* Add the encrypted key to the key map data file structure. */ + pg_tde_write_key_map_entry(newrlocator, &rel_key_data, principal_key); + LWLockRelease(lock_pk); + /* - * Add the encrypted key to the key map data file structure. + * It is fine to write the to WAL after writing to the file since we have + * not WAL logged the SMGR CREATE event either. */ - pg_tde_write_key_map_entry(newrlocator, &rel_key_data, principal_key, true); - LWLockRelease(lock_pk); + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(xlrec)); + XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_RELATION_KEY); return pg_tde_put_key_into_cache(newrlocator, &rel_key_data); } -static InternalKey * -pg_tde_create_local_key(const RelFileLocator *newrlocator, uint32 entry_type) +void +pg_tde_create_smgr_key_perm_redo(const RelFileLocator *newrlocator) { - InternalKey int_key; + InternalKey rel_key_data; + InternalKey *old_key; + TDEPrincipalKey *principal_key; + LWLock *lock_pk = tde_lwlock_enc_keys(); - pg_tde_generate_internal_key(&int_key, entry_type); + if ((old_key = pg_tde_get_key_from_file(newrlocator, TDE_KEY_TYPE_SMGR))) + { + pfree(old_key); + return; + } - return pg_tde_put_key_into_cache(newrlocator, &int_key); + pg_tde_generate_internal_key(&rel_key_data, TDE_KEY_TYPE_SMGR); + + LWLockAcquire(lock_pk, LW_EXCLUSIVE); + principal_key = GetPrincipalKey(newrlocator->dbOid, LW_EXCLUSIVE); + if (principal_key == NULL) + { + ereport(ERROR, + errmsg("principal key not configured"), + errhint("create one using pg_tde_set_key before using encrypted tables")); + } + + /* Add the encrypted key to the key map data file structure. */ + pg_tde_write_key_map_entry(newrlocator, &rel_key_data, principal_key); + LWLockRelease(lock_pk); } static void @@ -191,9 +226,14 @@ pg_tde_generate_internal_key(InternalKey *int_key, uint32 entry_type) if (!RAND_bytes(int_key->key, INTERNAL_KEY_LEN)) ereport(ERROR, - (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("could not generate internal key: %s", - ERR_error_string(ERR_get_error(), NULL)))); + errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate internal key: %s", + ERR_error_string(ERR_get_error(), NULL))); + if (!RAND_bytes(int_key->base_iv, INTERNAL_KEY_IV_LEN)) + ereport(ERROR, + errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate IV: %s", + ERR_error_string(ERR_get_error(), NULL))); } const char * @@ -228,8 +268,8 @@ pg_tde_create_wal_key(InternalKey *rel_key_data, const RelFileLocator *newrlocat if (principal_key == NULL) { ereport(ERROR, - (errmsg("principal key not configured"), - errhint("create one using pg_tde_set_server_principal_key before using encrypted WAL"))); + errmsg("principal key not configured"), + errhint("create one using pg_tde_set_server_key before using encrypted WAL")); } /* TODO: no need in generating key if TDE_KEY_TYPE_WAL_UNENCRYPTED */ @@ -238,7 +278,7 @@ pg_tde_create_wal_key(InternalKey *rel_key_data, const RelFileLocator *newrlocat /* * Add the encrypted key to the key map data file structure. */ - pg_tde_write_key_map_entry(newrlocator, rel_key_data, principal_key, false); + pg_tde_write_key_map_entry(newrlocator, rel_key_data, principal_key); LWLockRelease(tde_lwlock_enc_keys()); } @@ -249,9 +289,8 @@ pg_tde_create_wal_key(InternalKey *rel_key_data, const RelFileLocator *newrlocat void pg_tde_delete_tde_files(Oid dbOid) { - char db_map_path[MAXPGPATH] = {0}; + char db_map_path[MAXPGPATH]; - /* Set the file paths */ pg_tde_set_db_file_path(dbOid, db_map_path); /* Remove file without emitting any error */ @@ -259,11 +298,18 @@ pg_tde_delete_tde_files(Oid dbOid) } void -pg_tde_save_principal_key_redo(TDEPrincipalKeyInfo *principal_key_info) +pg_tde_save_principal_key_redo(const TDESignedPrincipalKeyInfo *signed_key_info) { + int map_fd; + off_t curr_pos; + char db_map_path[MAXPGPATH]; + + pg_tde_set_db_file_path(signed_key_info->data.databaseId, db_map_path); + LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); - pg_tde_save_principal_key(principal_key_info); + map_fd = pg_tde_open_file_write(db_map_path, signed_key_info, false, &curr_pos); + close(map_fd); LWLockRelease(tde_lwlock_enc_keys()); } @@ -277,18 +323,27 @@ pg_tde_save_principal_key_redo(TDEPrincipalKeyInfo *principal_key_info) * The caller must have an EXCLUSIVE LOCK on the files before calling this function. */ void -pg_tde_save_principal_key(TDEPrincipalKeyInfo *principal_key_info) +pg_tde_save_principal_key(const TDEPrincipalKey *principal_key, bool write_xlog) { - int map_fd = -1; + int map_fd; off_t curr_pos = 0; - char db_map_path[MAXPGPATH] = {0}; + char db_map_path[MAXPGPATH]; + TDESignedPrincipalKeyInfo signed_key_Info; - /* Set the file paths */ - pg_tde_set_db_file_path(principal_key_info->databaseId, db_map_path); + pg_tde_set_db_file_path(principal_key->keyInfo.databaseId, db_map_path); - ereport(DEBUG2, (errmsg("pg_tde_save_principal_key"))); + ereport(DEBUG2, errmsg("pg_tde_save_principal_key")); - map_fd = pg_tde_open_file_write(db_map_path, principal_key_info, true, &curr_pos); + pg_tde_sign_principal_key_info(&signed_key_Info, principal_key); + + if (write_xlog) + { + XLogBeginInsert(); + XLogRegisterData((char *) &signed_key_Info, sizeof(TDESignedPrincipalKeyInfo)); + XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_PRINCIPAL_KEY); + } + + map_fd = pg_tde_open_file_write(db_map_path, &signed_key_Info, true, &curr_pos); close(map_fd); } @@ -296,40 +351,52 @@ pg_tde_save_principal_key(TDEPrincipalKeyInfo *principal_key_info) * Write TDE file header to a TDE file. */ static int -pg_tde_file_header_write(const char *tde_filename, int fd, TDEPrincipalKeyInfo *principal_key_info, off_t *bytes_written) +pg_tde_file_header_write(const char *tde_filename, int fd, const TDESignedPrincipalKeyInfo *signed_key_info, off_t *bytes_written) { TDEFileHeader fheader; - Assert(principal_key_info); + Assert(signed_key_info); /* Create the header for this file. */ fheader.file_version = PG_TDE_FILEMAGIC; /* Fill in the data */ - fheader.principal_key_info = *principal_key_info; + fheader.signed_key_info = *signed_key_info; *bytes_written = pg_pwrite(fd, &fheader, TDE_FILE_HEADER_SIZE, 0); if (*bytes_written != TDE_FILE_HEADER_SIZE) { ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not write tde file \"%s\": %m", - tde_filename))); + errcode_for_file_access(), + errmsg("could not write tde file \"%s\": %m", tde_filename)); } if (pg_fsync(fd) != 0) { ereport(data_sync_elevel(ERROR), - (errcode_for_file_access(), - errmsg("could not fsync file \"%s\": %m", tde_filename))); + errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", tde_filename)); } - ereport(DEBUG2, - (errmsg("Wrote the header to %s", tde_filename))); + + ereport(DEBUG2, errmsg("Wrote the header to %s", tde_filename)); return fd; } +static void +pg_tde_sign_principal_key_info(TDESignedPrincipalKeyInfo *signed_key_info, const TDEPrincipalKey *principal_key) +{ + signed_key_info->data = principal_key->keyInfo; + + if (!RAND_bytes(signed_key_info->sign_iv, MAP_ENTRY_EMPTY_IV_SIZE)) + ereport(ERROR, + errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate iv for key map: %s", ERR_error_string(ERR_get_error(), NULL))); + + AesGcmEncrypt(principal_key->keyData, signed_key_info->sign_iv, (unsigned char *) &signed_key_info->data, sizeof(signed_key_info->data), NULL, 0, NULL, signed_key_info->aead_tag); +} + static void pg_tde_initialize_map_entry(TDEMapEntry *map_entry, const TDEPrincipalKey *principal_key, const RelFileLocator *rlocator, const InternalKey *rel_key_data) { @@ -340,8 +407,8 @@ pg_tde_initialize_map_entry(TDEMapEntry *map_entry, const TDEPrincipalKey *princ if (!RAND_bytes(map_entry->entry_iv, MAP_ENTRY_EMPTY_IV_SIZE)) ereport(ERROR, - (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("could not generate iv for key map: %s", ERR_error_string(ERR_get_error(), NULL)))); + errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate iv for key map: %s", ERR_error_string(ERR_get_error(), NULL))); AesGcmEncrypt(principal_key->keyData, map_entry->entry_iv, (unsigned char *) map_entry, offsetof(TDEMapEntry, enc_key), rel_key_data->key, INTERNAL_KEY_LEN, map_entry->enc_key.key, map_entry->aead_tag); } @@ -349,7 +416,7 @@ pg_tde_initialize_map_entry(TDEMapEntry *map_entry, const TDEPrincipalKey *princ /* * Based on the given arguments,write the entry into the key map file. */ -static off_t +static void pg_tde_write_one_map_entry(int fd, const TDEMapEntry *map_entry, off_t *offset, const char *db_map_path) { int bytes_written = 0; @@ -360,18 +427,17 @@ pg_tde_write_one_map_entry(int fd, const TDEMapEntry *map_entry, off_t *offset, if (bytes_written != MAP_ENTRY_SIZE) { ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not write tde map file \"%s\": %m", - db_map_path))); + errcode_for_file_access(), + errmsg("could not write tde map file \"%s\": %m", db_map_path)); } if (pg_fsync(fd) != 0) { ereport(data_sync_elevel(ERROR), - (errcode_for_file_access(), - errmsg("could not fsync file \"%s\": %m", db_map_path))); + errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", db_map_path)); } - return (*offset + bytes_written); + *offset += bytes_written; } /* @@ -387,22 +453,22 @@ pg_tde_write_one_map_entry(int fd, const TDEMapEntry *map_entry, off_t *offset, * concurrent in place updates leading to data conflicts. */ void -pg_tde_write_key_map_entry(const RelFileLocator *rlocator, InternalKey *rel_key_data, TDEPrincipalKey *principal_key, bool write_xlog) +pg_tde_write_key_map_entry(const RelFileLocator *rlocator, InternalKey *rel_key_data, TDEPrincipalKey *principal_key) { - char db_map_path[MAXPGPATH] = {0}; - int map_fd = -1; + char db_map_path[MAXPGPATH]; + int map_fd; off_t curr_pos = 0; - off_t prev_pos = 0; TDEMapEntry write_map_entry; + TDESignedPrincipalKeyInfo signed_key_Info; Assert(rlocator); - /* Set the file paths */ pg_tde_set_db_file_path(rlocator->dbOid, db_map_path); + pg_tde_sign_principal_key_info(&signed_key_Info, principal_key); + /* Open and validate file for basic correctness. */ - map_fd = pg_tde_open_file_write(db_map_path, &principal_key->keyInfo, false, &curr_pos); - prev_pos = curr_pos; + map_fd = pg_tde_open_file_write(db_map_path, &signed_key_Info, false, &curr_pos); /* * Read until we find an empty slot. Otherwise, read until end. This seems @@ -412,151 +478,61 @@ pg_tde_write_key_map_entry(const RelFileLocator *rlocator, InternalKey *rel_key_ while (1) { TDEMapEntry read_map_entry; - bool found; + off_t prev_pos = curr_pos; - prev_pos = curr_pos; - found = pg_tde_read_one_map_entry(map_fd, NULL, MAP_ENTRY_EMPTY, &read_map_entry, &curr_pos); + if (!pg_tde_read_one_map_entry(map_fd, &read_map_entry, &curr_pos)) + { + curr_pos = prev_pos; + break; + } - /* - * We either reach EOF or found an empty slot in the middle of the - * file - */ - if (prev_pos == curr_pos || found) + if (read_map_entry.flags == MAP_ENTRY_EMPTY) + { + curr_pos = prev_pos; break; + } } /* Initialize map entry and encrypt key */ pg_tde_initialize_map_entry(&write_map_entry, principal_key, rlocator, rel_key_data); - if (write_xlog) - { - XLogRelKey xlrec; - - xlrec.mapEntry = write_map_entry; - xlrec.pkInfo = principal_key->keyInfo; - - XLogBeginInsert(); - XLogRegisterData((char *) &xlrec, sizeof(xlrec)); - XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_RELATION_KEY); - } + /* Write the given entry at curr_pos; i.e. the free entry. */ + pg_tde_write_one_map_entry(map_fd, &write_map_entry, &curr_pos, db_map_path); - /* - * Write the given entry at the location pointed by prev_pos; i.e. the - * free entry - */ - curr_pos = prev_pos; - pg_tde_write_one_map_entry(map_fd, &write_map_entry, &prev_pos, db_map_path); - - /* Let's close the file. */ close(map_fd); - - /* Register the entry to be freed in case the transaction aborts */ - RegisterEntryForDeletion(rlocator, curr_pos, false); } /* - * Write and already encrypted entry to the key map. + * Mark relation map entry as free and overwrite the key * - * The caller must hold an exclusive lock on the map file to avoid - * concurrent in place updates leading to data conflicts. + * This fucntion is called by the pg_tde SMGR when storage is unlinked on + * transaction commit/abort. */ void -pg_tde_write_key_map_entry_redo(const TDEMapEntry *write_map_entry, TDEPrincipalKeyInfo *principal_key_info) +pg_tde_free_key_map_entry(const RelFileLocator *rlocator) { - char db_map_path[MAXPGPATH] = {0}; - int map_fd = -1; + char db_map_path[MAXPGPATH]; + File map_fd; off_t curr_pos = 0; - off_t prev_pos = 0; - - /* Set the file paths */ - pg_tde_set_db_file_path(principal_key_info->databaseId, db_map_path); - - LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); - - /* Open and validate file for basic correctness. */ - map_fd = pg_tde_open_file_write(db_map_path, principal_key_info, false, &curr_pos); - prev_pos = curr_pos; - - /* - * Read until we find an empty slot. Otherwise, read until end. This seems - * to be less frequent than vacuum. So let's keep this function here - * rather than overloading the vacuum process. - */ - while (1) - { - TDEMapEntry read_map_entry; - bool found; - - prev_pos = curr_pos; - found = pg_tde_read_one_map_entry(map_fd, NULL, MAP_ENTRY_EMPTY, &read_map_entry, &curr_pos); - - /* - * We either reach EOF or found an empty slot in the middle of the - * file - */ - if (prev_pos == curr_pos || found) - break; - } - /* - * Write the given entry at the location pointed by prev_pos; i.e. the - * free entry - */ - curr_pos = prev_pos; - pg_tde_write_one_map_entry(map_fd, write_map_entry, &prev_pos, db_map_path); - - /* Let's close the file. */ - close(map_fd); + Assert(rlocator); - LWLockRelease(tde_lwlock_enc_keys()); -} + pg_tde_set_db_file_path(rlocator->dbOid, db_map_path); -static bool -pg_tde_delete_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_t offset) -{ - File map_fd; - bool found = false; - off_t curr_pos = 0; + LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); /* Open and validate file for basic correctness. */ map_fd = pg_tde_open_file_write(db_map_path, NULL, false, &curr_pos); - /* - * If we need to delete an entry, we expect an offset value to the start - * of the entry to speed up the operation. Otherwise, we'd be sequentially - * scanning the entire map file. - */ - if (offset > 0) - { - curr_pos = lseek(map_fd, offset, SEEK_SET); - - if (curr_pos == -1) - { - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not seek in tde map file \"%s\": %m", - db_map_path))); - } - } - - /* - * Read until we find an empty slot. Otherwise, read until end. This seems - * to be less frequent than vacuum. So let's keep this function here - * rather than overloading the vacuum process. - */ while (1) { - TDEMapEntry read_map_entry; + TDEMapEntry map_entry; off_t prev_pos = curr_pos; - found = pg_tde_read_one_map_entry(map_fd, rlocator, MAP_ENTRY_VALID, &read_map_entry, &curr_pos); - - /* We've reached EOF */ - if (curr_pos == prev_pos) + if (!pg_tde_read_one_map_entry(map_fd, &map_entry, &curr_pos)) break; - /* We found a valid entry for the relation */ - if (found) + if (map_entry.flags != MAP_ENTRY_EMPTY && map_entry.spcOid == rlocator->spcOid && map_entry.relNumber == rlocator->relNumber) { TDEMapEntry empty_map_entry = { .flags = MAP_ENTRY_EMPTY, @@ -570,52 +546,9 @@ pg_tde_delete_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_t } } - /* Let's close the file. */ close(map_fd); - /* Return -1 indicating that no entry was removed */ - return found; -} - -/* - * Called when transaction is being completed; either committed or aborted. - * By default, when a transaction creates an entry, we mark it as MAP_ENTRY_VALID. - * Only during the abort phase of the transaction that we are proceed on with - * marking the entry as MAP_ENTRY_FREE. This optimistic strategy that assumes - * that transaction will commit more often then getting aborted avoids - * unnecessary locking. - * - * The offset allows us to simply seek to the desired location and mark the entry - * as MAP_ENTRY_FREE without needing any further processing. - */ -void -pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset) -{ - bool found; - char db_map_path[MAXPGPATH] = {0}; - - Assert(rlocator); - - /* Get the file paths */ - pg_tde_set_db_file_path(rlocator->dbOid, db_map_path); - - LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); - - /* Remove the map entry if found */ - found = pg_tde_delete_map_entry(rlocator, db_map_path, offset); - LWLockRelease(tde_lwlock_enc_keys()); - - if (!found) - { - ereport(WARNING, - (errcode(ERRCODE_NO_DATA_FOUND), - errmsg("could not find the required map entry for deletion of relation %d in tablespace %d in tde map file \"%s\": %m", - rlocator->relNumber, - rlocator->spcOid, - db_map_path))); - - } } /* @@ -626,7 +559,7 @@ pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset) * No error checking by this function. */ static File -keyrotation_init_file(TDEPrincipalKeyInfo *new_principal_key_info, char *rotated_filename, char *filename, off_t *curr_pos) +keyrotation_init_file(const TDESignedPrincipalKeyInfo *signed_key_info, char *rotated_filename, const char *filename, off_t *curr_pos) { /* * Set the new filenames for the key rotation process - temporary at the @@ -635,7 +568,7 @@ keyrotation_init_file(TDEPrincipalKeyInfo *new_principal_key_info, char *rotated snprintf(rotated_filename, MAXPGPATH, "%s.r", filename); /* Create file, truncate if the rotate file already exits */ - return pg_tde_open_file_write(rotated_filename, new_principal_key_info, true, curr_pos); + return pg_tde_open_file_write(rotated_filename, signed_key_info, true, curr_pos); } /* @@ -652,43 +585,35 @@ finalize_key_rotation(const char *path_old, const char *path_new) * Rotate keys and generates the WAL record for it. */ void -pg_tde_perform_rotate_key(TDEPrincipalKey *principal_key, TDEPrincipalKey *new_principal_key) +pg_tde_perform_rotate_key(TDEPrincipalKey *principal_key, TDEPrincipalKey *new_principal_key, bool write_xlog) { -#define OLD_PRINCIPAL_KEY 0 -#define NEW_PRINCIPAL_KEY 1 -#define PRINCIPAL_KEY_COUNT 2 + TDESignedPrincipalKeyInfo new_signed_key_info; + off_t old_curr_pos, + new_curr_pos; + int old_fd, + new_fd; + char old_path[MAXPGPATH], + new_path[MAXPGPATH]; - off_t curr_pos[PRINCIPAL_KEY_COUNT] = {0}; - int fd[PRINCIPAL_KEY_COUNT]; - char path[PRINCIPAL_KEY_COUNT][MAXPGPATH]; - off_t map_size; - XLogPrincipalKeyRotate *xlrec; - off_t xlrec_size; + pg_tde_sign_principal_key_info(&new_signed_key_info, new_principal_key); - pg_tde_set_db_file_path(principal_key->keyInfo.databaseId, path[OLD_PRINCIPAL_KEY]); + pg_tde_set_db_file_path(principal_key->keyInfo.databaseId, old_path); - fd[OLD_PRINCIPAL_KEY] = pg_tde_open_file_read(path[OLD_PRINCIPAL_KEY], &curr_pos[OLD_PRINCIPAL_KEY]); - fd[NEW_PRINCIPAL_KEY] = keyrotation_init_file(&new_principal_key->keyInfo, path[NEW_PRINCIPAL_KEY], path[OLD_PRINCIPAL_KEY], &curr_pos[NEW_PRINCIPAL_KEY]); + old_fd = pg_tde_open_file_read(old_path, &old_curr_pos); + new_fd = keyrotation_init_file(&new_signed_key_info, new_path, old_path, &new_curr_pos); /* Read all entries until EOF */ while (1) { InternalKey *rel_key_data; - off_t prev_pos[PRINCIPAL_KEY_COUNT]; TDEMapEntry read_map_entry, write_map_entry; RelFileLocator rloc; - bool found; - prev_pos[OLD_PRINCIPAL_KEY] = curr_pos[OLD_PRINCIPAL_KEY]; - found = pg_tde_read_one_map_entry(fd[OLD_PRINCIPAL_KEY], NULL, MAP_ENTRY_VALID, &read_map_entry, &curr_pos[OLD_PRINCIPAL_KEY]); - - /* We either reach EOF */ - if (prev_pos[OLD_PRINCIPAL_KEY] == curr_pos[OLD_PRINCIPAL_KEY]) + if (!pg_tde_read_one_map_entry(old_fd, &read_map_entry, &old_curr_pos)) break; - /* We didn't find a valid entry */ - if (found == false) + if (read_map_entry.flags == MAP_ENTRY_EMPTY) continue; rloc.spcOid = read_map_entry.spcOid; @@ -699,46 +624,44 @@ pg_tde_perform_rotate_key(TDEPrincipalKey *principal_key, TDEPrincipalKey *new_p rel_key_data = tde_decrypt_rel_key(principal_key, &read_map_entry); pg_tde_initialize_map_entry(&write_map_entry, new_principal_key, &rloc, rel_key_data); - /* Write the given entry at the location pointed by prev_pos */ - prev_pos[NEW_PRINCIPAL_KEY] = curr_pos[NEW_PRINCIPAL_KEY]; - curr_pos[NEW_PRINCIPAL_KEY] = pg_tde_write_one_map_entry(fd[NEW_PRINCIPAL_KEY], &write_map_entry, &prev_pos[NEW_PRINCIPAL_KEY], path[NEW_PRINCIPAL_KEY]); + pg_tde_write_one_map_entry(new_fd, &write_map_entry, &new_curr_pos, new_path); pfree(rel_key_data); } - close(fd[OLD_PRINCIPAL_KEY]); - - /* Let's calculate sizes */ - map_size = lseek(fd[NEW_PRINCIPAL_KEY], 0, SEEK_END); - xlrec_size = map_size + SizeoOfXLogPrincipalKeyRotate; - - /* palloc and fill in the structure */ - xlrec = (XLogPrincipalKeyRotate *) palloc(xlrec_size); + close(old_fd); + close(new_fd); - xlrec->databaseId = principal_key->keyInfo.databaseId; - xlrec->file_size = map_size; - - if (pg_pread(fd[NEW_PRINCIPAL_KEY], xlrec->buff, xlrec->file_size, 0) == -1) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not write WAL for key rotation: %m"))); - - close(fd[NEW_PRINCIPAL_KEY]); - - /* Insert the XLog record */ - XLogBeginInsert(); - XLogRegisterData((char *) xlrec, xlrec_size); - XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ROTATE_KEY); + /* + * Do the final steps - replace the current _map with the file with new + * data + */ + finalize_key_rotation(old_path, new_path); - /* Do the final steps */ - finalize_key_rotation(path[OLD_PRINCIPAL_KEY], path[NEW_PRINCIPAL_KEY]); + /* + * We do WAL writes past the event ("the write behind logging") rather + * than before ("the write ahead") because we need logging here only for + * replication purposes. The rotation results in data written and fsynced + * to disk. Which in most cases would happen way before it's written to + * the WAL disk file. As WAL will be flushed at the end of the + * transaction, on its commit, hence after this function returns (there is + * also a bg writer, but the commit is what is guaranteed). And it makes + * sense to replicate the event only after its effect has been + * successfully applied to the source. + */ + if (write_xlog) + { + XLogPrincipalKeyRotate xlrec; - /* Free up the palloc'ed data */ - pfree(xlrec); + xlrec.databaseId = principal_key->keyInfo.databaseId; + xlrec.keyringId = principal_key->keyInfo.keyringId; + memcpy(xlrec.keyName, new_principal_key->keyInfo.name, sizeof(new_principal_key->keyInfo.name)); -#undef OLD_PRINCIPAL_KEY -#undef NEW_PRINCIPAL_KEY -#undef PRINCIPAL_KEY_COUNT + /* Insert the XLog record */ + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(XLogPrincipalKeyRotate)); + XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ROTATE_PRINCIPAL_KEY); + } } /* @@ -748,53 +671,51 @@ void pg_tde_write_map_keydata_file(off_t file_size, char *file_data) { TDEFileHeader *fheader; + char db_map_path[MAXPGPATH]; char path_new[MAXPGPATH]; int fd_new; off_t curr_pos = 0; - char db_map_path[MAXPGPATH] = {0}; - bool is_err = false; /* Let's get the header. Buff should start with the map file header. */ fheader = (TDEFileHeader *) file_data; - /* Set the file paths */ - pg_tde_set_db_file_path(fheader->principal_key_info.databaseId, db_map_path); + pg_tde_set_db_file_path(fheader->signed_key_info.data.databaseId, db_map_path); /* Initialize the new file and set the name */ - fd_new = keyrotation_init_file(&fheader->principal_key_info, path_new, db_map_path, &curr_pos); + fd_new = keyrotation_init_file(&fheader->signed_key_info, path_new, db_map_path, &curr_pos); if (pg_pwrite(fd_new, file_data, file_size, 0) != file_size) { ereport(WARNING, - (errcode_for_file_access(), - errmsg("could not write tde file \"%s\": %m", path_new))); - is_err = true; - goto FINALIZE; + errcode_for_file_access(), + errmsg("could not write tde file \"%s\": %m", path_new)); + close(fd_new); + return; } + if (pg_fsync(fd_new) != 0) { ereport(WARNING, - (errcode_for_file_access(), - errmsg("could not fsync file \"%s\": %m", path_new))); - is_err = true; - goto FINALIZE; + errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", path_new)); + close(fd_new); + return; } -FINALIZE: close(fd_new); - if (!is_err) - finalize_key_rotation(db_map_path, path_new); + finalize_key_rotation(db_map_path, path_new); } -/* It's called by seg_write inside crit section so no pallocs, hence +/* + * It's called by seg_write inside crit section so no pallocs, hence * needs keyfile_path */ void pg_tde_wal_last_key_set_lsn(XLogRecPtr lsn, const char *keyfile_path) { LWLock *lock_pk = tde_lwlock_enc_keys(); - int fd = -1; + int fd; off_t read_pos, write_pos, last_key_idx; @@ -809,8 +730,8 @@ pg_tde_wal_last_key_set_lsn(XLogRecPtr lsn, const char *keyfile_path) if (pg_pwrite(fd, &lsn, sizeof(XLogRecPtr), write_pos) != sizeof(XLogRecPtr)) { ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not write tde key data file: %m"))); + errcode_for_file_access(), + errmsg("could not write tde key data file: %m")); } /* @@ -826,8 +747,8 @@ pg_tde_wal_last_key_set_lsn(XLogRecPtr lsn, const char *keyfile_path) if (pg_pread(fd, &prev_map_entry, MAP_ENTRY_SIZE, prev_key_pos) != MAP_ENTRY_SIZE) { ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not read previous WAL key: %m"))); + errcode_for_file_access(), + errmsg("could not read previous WAL key: %m")); } if (prev_map_entry.enc_key.start_lsn >= lsn) @@ -837,8 +758,8 @@ pg_tde_wal_last_key_set_lsn(XLogRecPtr lsn, const char *keyfile_path) if (pg_pwrite(fd, &prev_map_entry, MAP_ENTRY_SIZE, prev_key_pos) != MAP_ENTRY_SIZE) { ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not write invalidated key: %m"))); + errcode_for_file_access(), + errmsg("could not write invalidated key: %m")); } } } @@ -846,8 +767,8 @@ pg_tde_wal_last_key_set_lsn(XLogRecPtr lsn, const char *keyfile_path) if (pg_fsync(fd) != 0) { ereport(data_sync_elevel(ERROR), - (errcode_for_file_access(), - errmsg("could not fsync file: %m"))); + errcode_for_file_access(), + errmsg("could not fsync file: %m")); } LWLockRelease(lock_pk); @@ -862,7 +783,7 @@ pg_tde_wal_last_key_set_lsn(XLogRecPtr lsn, const char *keyfile_path) * is raised. */ static int -pg_tde_open_file_write(const char *tde_filename, TDEPrincipalKeyInfo *principal_key_info, bool truncate, off_t *curr_pos) +pg_tde_open_file_write(const char *tde_filename, const TDESignedPrincipalKeyInfo *signed_key_info, bool truncate, off_t *curr_pos) { int fd; TDEFileHeader fheader; @@ -877,8 +798,8 @@ pg_tde_open_file_write(const char *tde_filename, TDEPrincipalKeyInfo *principal_ pg_tde_file_header_read(tde_filename, fd, &fheader, &bytes_read); /* In case it's a new file, let's add the header now. */ - if (bytes_read == 0 && principal_key_info) - pg_tde_file_header_write(tde_filename, fd, principal_key_info, &bytes_written); + if (bytes_read == 0 && signed_key_info) + pg_tde_file_header_write(tde_filename, fd, signed_key_info, &bytes_written); *curr_pos = bytes_read + bytes_written; return fd; @@ -893,15 +814,14 @@ pg_tde_open_file_write(const char *tde_filename, TDEPrincipalKeyInfo *principal_ static InternalKey * pg_tde_get_key_from_file(const RelFileLocator *rlocator, uint32 key_type) { - TDEMapEntry *map_entry; + TDEMapEntry map_entry; TDEPrincipalKey *principal_key; LWLock *lock_pk = tde_lwlock_enc_keys(); - char db_map_path[MAXPGPATH] = {0}; + char db_map_path[MAXPGPATH]; InternalKey *rel_key; Assert(rlocator); - /* Get the file paths */ pg_tde_set_db_file_path(rlocator->dbOid, db_map_path); if (access(db_map_path, F_OK) == -1) @@ -909,10 +829,7 @@ pg_tde_get_key_from_file(const RelFileLocator *rlocator, uint32 key_type) LWLockAcquire(lock_pk, LW_SHARED); - /* Read the map entry and get the index of the relation key */ - map_entry = pg_tde_find_map_entry(rlocator, key_type, db_map_path); - - if (map_entry == NULL) + if (!pg_tde_find_map_entry(rlocator, key_type, db_map_path, &map_entry)) { LWLockRelease(lock_pk); return NULL; @@ -933,10 +850,10 @@ pg_tde_get_key_from_file(const RelFileLocator *rlocator, uint32 key_type) principal_key = GetPrincipalKey(rlocator->dbOid, LW_SHARED); if (principal_key == NULL) ereport(ERROR, - (errmsg("principal key not configured"), - errhint("create one using pg_tde_set_principal_key before using encrypted tables"))); + errmsg("principal key not configured"), + errhint("create one using pg_tde_set_key before using encrypted tables")); - rel_key = tde_decrypt_rel_key(principal_key, map_entry); + rel_key = tde_decrypt_rel_key(principal_key, &map_entry); LWLockRelease(lock_pk); @@ -944,45 +861,39 @@ pg_tde_get_key_from_file(const RelFileLocator *rlocator, uint32 key_type) } /* - * Returns the index of the read map if we find a valid match; e.g. flags is set to - * MAP_ENTRY_VALID and the relNumber and spcOid matches the one provided in rlocator. + * Returns true if we find a valid match; e.g. flags is not set to + * MAP_ENTRY_EMPTY and the relNumber and spcOid matches the one provided in + * rlocator. */ -static TDEMapEntry * -pg_tde_find_map_entry(const RelFileLocator *rlocator, uint32 key_type, char *db_map_path) +static bool +pg_tde_find_map_entry(const RelFileLocator *rlocator, uint32 key_type, char *db_map_path, TDEMapEntry *map_entry) { File map_fd; - TDEMapEntry *map_entry = palloc_object(TDEMapEntry); off_t curr_pos = 0; + bool found = false; + + Assert(rlocator != NULL); map_fd = pg_tde_open_file_read(db_map_path, &curr_pos); - /* - * Read until we find an empty slot. Otherwise, read until end. This seems - * to be less frequent than vacuum. So let's keep this function here - * rather than overloading the vacuum process. - */ - while (1) + while (pg_tde_read_one_map_entry(map_fd, map_entry, &curr_pos)) { - bool found; - off_t prev_pos = curr_pos; - - found = pg_tde_read_one_map_entry(map_fd, rlocator, key_type, map_entry, &curr_pos); - - /* We've reached EOF */ - if (curr_pos == prev_pos) - { - close(map_fd); - pfree(map_entry); - return NULL; - } - - /* We found a valid entry for the relation */ - if (found) + if ((map_entry->flags & key_type) && map_entry->spcOid == rlocator->spcOid && map_entry->relNumber == rlocator->relNumber) { - close(map_fd); - return map_entry; + found = true; + break; } } + + close(map_fd); + + return found; +} + +bool +pg_tde_verify_principal_key_info(TDESignedPrincipalKeyInfo *signed_key_info, const TDEPrincipalKey *principal_key) +{ + return AesGcmDecrypt(principal_key->keyData, signed_key_info->sign_iv, (unsigned char *) &signed_key_info->data, sizeof(signed_key_info->data), NULL, 0, NULL, signed_key_info->aead_tag); } /* @@ -1001,7 +912,7 @@ tde_decrypt_rel_key(TDEPrincipalKey *principal_key, TDEMapEntry *map_entry) if (!AesGcmDecrypt(principal_key->keyData, map_entry->entry_iv, (unsigned char *) map_entry, offsetof(TDEMapEntry, enc_key), map_entry->enc_key.key, INTERNAL_KEY_LEN, rel_key_data->key, map_entry->aead_tag)) ereport(ERROR, - (errmsg("Failed to decrypt key, incorrect principal key or corrupted key file"))); + errmsg("Failed to decrypt key, incorrect principal key or corrupted key file")); return rel_key_data; @@ -1046,9 +957,8 @@ pg_tde_open_file_basic(const char *tde_filename, int fileFlags, bool ignore_miss if (fd < 0 && !(errno == ENOENT && ignore_missing == true)) { ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not open tde file \"%s\": %m", - tde_filename))); + errcode_for_file_access(), + errmsg("could not open tde file \"%s\": %m", tde_filename)); } return fd; @@ -1075,34 +985,24 @@ pg_tde_file_header_read(const char *tde_filename, int fd, TDEFileHeader *fheader /* Corrupt file */ close(fd); ereport(FATAL, - (errcode_for_file_access(), - errmsg("TDE map file \"%s\" is corrupted: %m", - tde_filename))); + errcode_for_file_access(), + errmsg("TDE map file \"%s\" is corrupted: %m", tde_filename)); } } /* - * Returns true if a valid map entry if found. Otherwise, it only increments - * the offset and returns false. If the same offset value is set, it indicates - * to the caller that nothing was read. - * - * If a non-NULL rlocator is provided, the function compares the read value - * against the relNumber of rlocator. It sets found accordingly. - * - * The caller is reponsible for identifying that we have reached EOF by - * comparing old and new value of the offset. + * Returns true if a map entry if found or false if we have reached the end of + * the file. */ static bool -pg_tde_read_one_map_entry(File map_file, const RelFileLocator *rlocator, int flags, TDEMapEntry *map_entry, off_t *offset) +pg_tde_read_one_map_entry(int map_file, TDEMapEntry *map_entry, off_t *offset) { - bool found; off_t bytes_read = 0; Assert(map_entry); Assert(offset); - /* Read the entry at the given offset */ bytes_read = pg_pread(map_file, map_entry, MAP_ENTRY_SIZE, *offset); /* We've reached the end of the file. */ @@ -1111,73 +1011,46 @@ pg_tde_read_one_map_entry(File map_file, const RelFileLocator *rlocator, int fla *offset += bytes_read; - /* We found a valid entry */ - found = map_entry->flags & flags; - - /* If a valid rlocator is provided, let's compare and set found value */ - if (rlocator != NULL) - found &= map_entry->spcOid == rlocator->spcOid && map_entry->relNumber == rlocator->relNumber; - - return found; + return true; } /* * TODO: Unify with pg_tde_read_one_map_entry() */ -static TDEMapEntry * -pg_tde_read_one_map_entry2(int fd, int32 key_index, TDEPrincipalKey *principal_key) +static void +pg_tde_read_one_map_entry2(int fd, int32 key_index, TDEMapEntry *map_entry, Oid databaseId) { - TDEMapEntry *map_entry; - off_t read_pos = 0; + off_t read_pos; /* Calculate the reading position in the file. */ - read_pos += (key_index * MAP_ENTRY_SIZE) + TDE_FILE_HEADER_SIZE; - - /* Check if the file has a valid key */ - if ((read_pos + MAP_ENTRY_SIZE) > lseek(fd, 0, SEEK_END)) - { - char db_map_path[MAXPGPATH] = {0}; - - pg_tde_set_db_file_path(principal_key->keyInfo.databaseId, db_map_path); - ereport(FATAL, - (errcode(ERRCODE_NO_DATA_FOUND), - errmsg("could not find the required key at index %d in tde data file \"%s\": %m", - key_index, db_map_path))); - } - - /* Allocate and fill in the structure */ - map_entry = palloc_object(TDEMapEntry); + read_pos = TDE_FILE_HEADER_SIZE + key_index * MAP_ENTRY_SIZE; /* Read the encrypted key */ if (pg_pread(fd, map_entry, MAP_ENTRY_SIZE, read_pos) != MAP_ENTRY_SIZE) { - char db_map_path[MAXPGPATH] = {0}; + char db_map_path[MAXPGPATH]; - pg_tde_set_db_file_path(principal_key->keyInfo.databaseId, db_map_path); + pg_tde_set_db_file_path(databaseId, db_map_path); ereport(FATAL, - (errcode_for_file_access(), - errmsg("could not read key at index %d in tde key data file \"%s\": %m", - key_index, db_map_path))); + errcode_for_file_access(), + errmsg("could not find the required key at index %d in tde data file \"%s\": %m", + key_index, db_map_path)); } - - return map_entry; } - /* * Get the principal key from the map file. The caller must hold * a LW_SHARED or higher lock on files before calling this function. */ -TDEPrincipalKeyInfo * +TDESignedPrincipalKeyInfo * pg_tde_get_principal_key_info(Oid dbOid) { - int fd = -1; + char db_map_path[MAXPGPATH]; + int fd; TDEFileHeader fheader; - TDEPrincipalKeyInfo *principal_key_info = NULL; + TDESignedPrincipalKeyInfo *signed_key_info = NULL; off_t bytes_read = 0; - char db_map_path[MAXPGPATH] = {0}; - /* Set the file paths */ pg_tde_set_db_file_path(dbOid, db_map_path); /* @@ -1200,11 +1073,11 @@ pg_tde_get_principal_key_info(Oid dbOid) */ if (bytes_read > 0) { - principal_key_info = palloc_object(TDEPrincipalKeyInfo); - *principal_key_info = fheader.principal_key_info; + signed_key_info = palloc_object(TDESignedPrincipalKeyInfo); + *signed_key_info = fheader.signed_key_info; } - return principal_key_info; + return signed_key_info; } /* @@ -1294,17 +1167,16 @@ InternalKey * pg_tde_read_last_wal_key(void) { RelFileLocator rlocator = GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID); - char db_map_path[MAXPGPATH] = {0}; + char db_map_path[MAXPGPATH]; off_t read_pos = 0; LWLock *lock_pk = tde_lwlock_enc_keys(); TDEPrincipalKey *principal_key; - int fd = -1; - int file_idx = 0; - TDEMapEntry *map_entry; + int fd; + int file_idx; + TDEMapEntry map_entry; InternalKey *rel_key_data; off_t fsize; - LWLockAcquire(lock_pk, LW_EXCLUSIVE); principal_key = GetPrincipalKey(rlocator.dbOid, LW_EXCLUSIVE); if (principal_key == NULL) @@ -1325,17 +1197,11 @@ pg_tde_read_last_wal_key(void) } file_idx = ((fsize - TDE_FILE_HEADER_SIZE) / MAP_ENTRY_SIZE) - 1; - map_entry = pg_tde_read_one_map_entry2(fd, file_idx, principal_key); - if (!map_entry) - { - LWLockRelease(lock_pk); - return NULL; - } + pg_tde_read_one_map_entry2(fd, file_idx, &map_entry, rlocator.dbOid); - rel_key_data = tde_decrypt_rel_key(principal_key, map_entry); + rel_key_data = tde_decrypt_rel_key(principal_key, &map_entry); LWLockRelease(lock_pk); close(fd); - pfree(map_entry); return rel_key_data; } @@ -1345,11 +1211,11 @@ WALKeyCacheRec * pg_tde_fetch_wal_keys(XLogRecPtr start_lsn) { RelFileLocator rlocator = GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID); - char db_map_path[MAXPGPATH] = {0}; + char db_map_path[MAXPGPATH]; off_t read_pos = 0; LWLock *lock_pk = tde_lwlock_enc_keys(); TDEPrincipalKey *principal_key; - int fd = -1; + int fd; int keys_count; WALKeyCacheRec *return_wal_rec = NULL; @@ -1391,26 +1257,27 @@ pg_tde_fetch_wal_keys(XLogRecPtr start_lsn) for (int file_idx = 0; file_idx < keys_count; file_idx++) { - TDEMapEntry *map_entry = pg_tde_read_one_map_entry2(fd, file_idx, principal_key); + TDEMapEntry map_entry; + + pg_tde_read_one_map_entry2(fd, file_idx, &map_entry, rlocator.dbOid); /* * Skip new (just created but not updated by write) and invalid keys */ - if (map_entry->enc_key.start_lsn != InvalidXLogRecPtr && - WALKeyIsValid(&map_entry->enc_key) && - map_entry->enc_key.start_lsn >= start_lsn) + if (map_entry.enc_key.start_lsn != InvalidXLogRecPtr && + WALKeyIsValid(&map_entry.enc_key) && + map_entry.enc_key.start_lsn >= start_lsn) { - InternalKey *rel_key_data = tde_decrypt_rel_key(principal_key, map_entry); + InternalKey *rel_key_data = tde_decrypt_rel_key(principal_key, &map_entry); InternalKey *cached_key = pg_tde_put_key_into_cache(&rlocator, rel_key_data); WALKeyCacheRec *wal_rec; pfree(rel_key_data); - wal_rec = pg_tde_add_wal_key_to_cache(cached_key, map_entry->enc_key.start_lsn); + wal_rec = pg_tde_add_wal_key_to_cache(cached_key, map_entry.enc_key.start_lsn); if (!return_wal_rec) return_wal_rec = wal_rec; } - pfree(map_entry); } LWLockRelease(lock_pk); close(fd); @@ -1444,7 +1311,7 @@ pg_tde_add_wal_key_to_cache(InternalKey *cached_key, XLogRecPtr start_lsn) else { tde_wal_key_last_rec->next = wal_rec; - tde_wal_key_last_rec->end_lsn = wal_rec->start_lsn - 1; + tde_wal_key_last_rec->end_lsn = wal_rec->start_lsn; tde_wal_key_last_rec = wal_rec; } diff --git a/contrib/pg_tde/src/access/pg_tde_xlog.c b/contrib/pg_tde/src/access/pg_tde_xlog.c index 9eea9e23a33da..9d817036bd8ef 100644 --- a/contrib/pg_tde/src/access/pg_tde_xlog.c +++ b/contrib/pg_tde/src/access/pg_tde_xlog.c @@ -30,10 +30,8 @@ static void tdeheap_rmgr_redo(XLogReaderState *record); static void tdeheap_rmgr_desc(StringInfo buf, XLogReaderState *record); static const char *tdeheap_rmgr_identify(uint8 info); -#define RM_TDERMGR_NAME "test_tdeheap_custom_rmgr" - static const RmgrData tdeheap_rmgr = { - .rm_name = RM_TDERMGR_NAME, + .rm_name = "pg_tde", .rm_redo = tdeheap_rmgr_redo, .rm_desc = tdeheap_rmgr_desc, .rm_identify = tdeheap_rmgr_identify, @@ -45,9 +43,6 @@ RegisterTdeRmgr(void) RegisterCustomRmgr(RM_TDERMGR_ID, &tdeheap_rmgr); } -/* - * TDE fork XLog - */ static void tdeheap_rmgr_redo(XLogReaderState *record) { @@ -57,40 +52,31 @@ tdeheap_rmgr_redo(XLogReaderState *record) { XLogRelKey *xlrec = (XLogRelKey *) XLogRecGetData(record); - pg_tde_write_key_map_entry_redo(&xlrec->mapEntry, &xlrec->pkInfo); + pg_tde_create_smgr_key_perm_redo(&xlrec->rlocator); } else if (info == XLOG_TDE_ADD_PRINCIPAL_KEY) { - TDEPrincipalKeyInfo *mkey = (TDEPrincipalKeyInfo *) XLogRecGetData(record); + TDESignedPrincipalKeyInfo *mkey = (TDESignedPrincipalKeyInfo *) XLogRecGetData(record); pg_tde_save_principal_key_redo(mkey); } - else if (info == XLOG_TDE_EXTENSION_INSTALL_KEY) + else if (info == XLOG_TDE_ROTATE_PRINCIPAL_KEY) { - XLogExtensionInstall *xlrec = (XLogExtensionInstall *) XLogRecGetData(record); + XLogPrincipalKeyRotate *xlrec = (XLogPrincipalKeyRotate *) XLogRecGetData(record); - extension_install_redo(xlrec); + xl_tde_perform_rotate_key(xlrec); } - - else if (info == XLOG_TDE_ADD_KEY_PROVIDER_KEY) + else if (info == XLOG_TDE_WRITE_KEY_PROVIDER) { - KeyringProviderXLRecord *xlrec = (KeyringProviderXLRecord *) XLogRecGetData(record); + KeyringProviderRecordInFile *xlrec = (KeyringProviderRecordInFile *) XLogRecGetData(record); redo_key_provider_info(xlrec); } - - else if (info == XLOG_TDE_ROTATE_KEY) - { - XLogPrincipalKeyRotate *xlrec = (XLogPrincipalKeyRotate *) XLogRecGetData(record); - - xl_tde_perform_rotate_key(xlrec); - } - - else if (info == XLOG_TDE_FREE_MAP_ENTRY) + else if (info == XLOG_TDE_INSTALL_EXTENSION) { - RelFileLocator *xlrec = (RelFileLocator *) XLogRecGetData(record); + XLogExtensionInstall *xlrec = (XLogExtensionInstall *) XLogRecGetData(record); - pg_tde_free_key_map_entry(xlrec, 0); + extension_install_redo(xlrec); } else { @@ -107,45 +93,50 @@ tdeheap_rmgr_desc(StringInfo buf, XLogReaderState *record) { XLogRelKey *xlrec = (XLogRelKey *) XLogRecGetData(record); - appendStringInfo(buf, "add tde internal key for relation %u/%u", xlrec->pkInfo.databaseId, xlrec->mapEntry.relNumber); + appendStringInfo(buf, "rel: %u/%u/%u", xlrec->rlocator.spcOid, xlrec->rlocator.dbOid, xlrec->rlocator.relNumber); } - if (info == XLOG_TDE_ADD_PRINCIPAL_KEY) + else if (info == XLOG_TDE_ADD_PRINCIPAL_KEY) { TDEPrincipalKeyInfo *xlrec = (TDEPrincipalKeyInfo *) XLogRecGetData(record); - appendStringInfo(buf, "add tde principal key for db %u", xlrec->databaseId); + appendStringInfo(buf, "db: %u", xlrec->databaseId); } - if (info == XLOG_TDE_EXTENSION_INSTALL_KEY) + else if (info == XLOG_TDE_ROTATE_PRINCIPAL_KEY) { - XLogExtensionInstall *xlrec = (XLogExtensionInstall *) XLogRecGetData(record); + XLogPrincipalKeyRotate *xlrec = (XLogPrincipalKeyRotate *) XLogRecGetData(record); - appendStringInfo(buf, "tde extension install for db %u", xlrec->database_id); + appendStringInfo(buf, "db: %u", xlrec->databaseId); } - if (info == XLOG_TDE_ROTATE_KEY) + else if (info == XLOG_TDE_WRITE_KEY_PROVIDER) { - XLogPrincipalKeyRotate *xlrec = (XLogPrincipalKeyRotate *) XLogRecGetData(record); + KeyringProviderRecordInFile *xlrec = (KeyringProviderRecordInFile *) XLogRecGetData(record); - appendStringInfo(buf, "rotate principal key for %u", xlrec->databaseId); + appendStringInfo(buf, "db: %u, provider id: %d", xlrec->database_id, xlrec->provider.provider_id); } - if (info == XLOG_TDE_ADD_KEY_PROVIDER_KEY) + else if (info == XLOG_TDE_INSTALL_EXTENSION) { - KeyringProviderXLRecord *xlrec = (KeyringProviderXLRecord *) XLogRecGetData(record); + XLogExtensionInstall *xlrec = (XLogExtensionInstall *) XLogRecGetData(record); - appendStringInfo(buf, "add key provider %s for %u", xlrec->provider.provider_name, xlrec->database_id); + appendStringInfo(buf, "db: %u", xlrec->database_id); } } static const char * tdeheap_rmgr_identify(uint8 info) { - if ((info & ~XLR_INFO_MASK) == XLOG_TDE_ADD_RELATION_KEY) - return "XLOG_TDE_ADD_RELATION_KEY"; - - if ((info & ~XLR_INFO_MASK) == XLOG_TDE_ADD_PRINCIPAL_KEY) - return "XLOG_TDE_ADD_PRINCIPAL_KEY"; - - if ((info & ~XLR_INFO_MASK) == XLOG_TDE_EXTENSION_INSTALL_KEY) - return "XLOG_TDE_EXTENSION_INSTALL_KEY"; - - return NULL; + switch (info & ~XLR_INFO_MASK) + { + case XLOG_TDE_ADD_RELATION_KEY: + return "ADD_RELATION_KEY"; + case XLOG_TDE_ADD_PRINCIPAL_KEY: + return "ADD_PRINCIPAL_KEY"; + case XLOG_TDE_ROTATE_PRINCIPAL_KEY: + return "ROTATE_PRINCIPAL_KEY"; + case XLOG_TDE_WRITE_KEY_PROVIDER: + return "WRITE_KEY_PROVIDER"; + case XLOG_TDE_INSTALL_EXTENSION: + return "INSTALL_EXTENSION"; + default: + return NULL; + } } diff --git a/contrib/pg_tde/src/access/pg_tde_xlog_encrypt.c b/contrib/pg_tde/src/access/pg_tde_xlog_smgr.c similarity index 78% rename from contrib/pg_tde/src/access/pg_tde_xlog_encrypt.c rename to contrib/pg_tde/src/access/pg_tde_xlog_smgr.c index c41af6ae45b09..f6b0a69cf0a51 100644 --- a/contrib/pg_tde/src/access/pg_tde_xlog_encrypt.c +++ b/contrib/pg_tde/src/access/pg_tde_xlog_smgr.c @@ -1,11 +1,11 @@ /*------------------------------------------------------------------------- * - * pg_tde_xlog_encrypt.c + * pg_tde_xlog_smgr.c * Encrypted XLog storage manager * * * IDENTIFICATION - * src/access/pg_tde_xlog_encrypt.c + * src/access/pg_tde_xlog_smgr.c * *------------------------------------------------------------------------- */ @@ -25,7 +25,7 @@ #include "utils/memutils.h" #include "access/pg_tde_tdemap.h" -#include "access/pg_tde_xlog_encrypt.h" +#include "access/pg_tde_xlog_smgr.h" #include "catalog/tde_global_space.h" #include "encryption/enc_tde.h" @@ -35,7 +35,7 @@ #include "port/atomics.h" #endif -static void SetXLogPageIVPrefix(TimeLineID tli, XLogRecPtr lsn, char *iv_prefix); +static void CalcXLogPageIVPrefix(TimeLineID tli, XLogRecPtr lsn, const unsigned char *base_iv, char *iv_prefix); static ssize_t tdeheap_xlog_seg_read(int fd, void *buf, size_t count, off_t offset, TimeLineID tli, XLogSegNo segno, int segSize); static ssize_t tdeheap_xlog_seg_write(int fd, const void *buf, size_t count, @@ -164,7 +164,7 @@ static ssize_t TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset, TimeLineID tli, XLogSegNo segno) { - char iv_prefix[16] = {0,}; + char iv_prefix[16]; InternalKey *key = &EncryptionKey; char *enc_buff = EncryptionState->segBuf; @@ -175,7 +175,7 @@ TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset, count, offset, offset, LSN_FORMAT_ARGS(segno), LSN_FORMAT_ARGS(key->start_lsn)); #endif - SetXLogPageIVPrefix(tli, segno, iv_prefix); + CalcXLogPageIVPrefix(tli, segno, key->base_iv, iv_prefix); PG_TDE_ENCRYPT_DATA(iv_prefix, offset, (char *) buf, count, enc_buff, key, &EncryptionCryptCtx); @@ -192,21 +192,30 @@ TDEXLogSmgrInit(void) /* TODO: move to the separate func, it's not an SMGR init */ InternalKey *key = pg_tde_read_last_wal_key(); - /* TDOO: clean-up this mess */ - if ((!key && EncryptXLog) || (key && - ((key->rel_type & TDE_KEY_TYPE_WAL_ENCRYPTED && !EncryptXLog) || - (key->rel_type & TDE_KEY_TYPE_WAL_UNENCRYPTED && EncryptXLog)))) + /* + * Always generate a new key on starting PostgreSQL to protect against + * attacks on CTR ciphers based on comparing the WAL generated by two + * divergent copies of the same cluster. + */ + if (EncryptXLog) + { + pg_tde_create_wal_key(&EncryptionKey, &GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), + TDE_KEY_TYPE_WAL_ENCRYPTED); + } + else if (key && key->rel_type & TDE_KEY_TYPE_WAL_ENCRYPTED) { pg_tde_create_wal_key(&EncryptionKey, &GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), - (EncryptXLog ? TDE_KEY_TYPE_WAL_ENCRYPTED : TDE_KEY_TYPE_WAL_UNENCRYPTED)); + TDE_KEY_TYPE_WAL_UNENCRYPTED); } else if (key) { EncryptionKey = *key; - pfree(key); pg_atomic_write_u64(&EncryptionState->enc_key_lsn, EncryptionKey.start_lsn); } + if (key) + pfree(key); + pg_tde_set_db_file_path(GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID).dbOid, EncryptionState->db_map_path); #endif @@ -251,13 +260,8 @@ tdeheap_xlog_seg_read(int fd, void *buf, size_t count, off_t offset, TimeLineID tli, XLogSegNo segno, int segSize) { ssize_t readsz; - char iv_prefix[16] = {0,}; WALKeyCacheRec *keys = pg_tde_get_wal_cache_keys(); - XLogRecPtr write_key_lsn = 0; - WALKeyCacheRec *curr_key = NULL; - off_t dec_off = 0; - off_t dec_end = 0; - size_t dec_sz = 0; + XLogRecPtr write_key_lsn; XLogRecPtr data_start; XLogRecPtr data_end; @@ -279,7 +283,6 @@ tdeheap_xlog_seg_read(int fd, void *buf, size_t count, off_t offset, #ifndef FRONTEND write_key_lsn = pg_atomic_read_u64(&EncryptionState->enc_key_lsn); -#endif if (write_key_lsn != 0) { @@ -296,8 +299,7 @@ tdeheap_xlog_seg_read(int fd, void *buf, size_t count, off_t offset, keys = pg_tde_get_wal_cache_keys(); } } - - SetXLogPageIVPrefix(tli, segno, iv_prefix); +#endif XLogSegNoOffsetToRecPtr(segno, offset, segSize, data_start); XLogSegNoOffsetToRecPtr(segno, offset + count, segSize, data_end); @@ -306,8 +308,7 @@ tdeheap_xlog_seg_read(int fd, void *buf, size_t count, off_t offset, * TODO: this is higly ineffective. We should get rid of linked list and * search from the last key as this is what the walsender is useing. */ - curr_key = keys; - while (curr_key) + for (WALKeyCacheRec *curr_key = keys; curr_key != NULL; curr_key = curr_key->next) { #ifdef TDE_XLOG_DEBUG elog(DEBUG1, "WAL key %X/%X-%X/%X, encrypted: %s", @@ -323,10 +324,14 @@ tdeheap_xlog_seg_read(int fd, void *buf, size_t count, off_t offset, * Check if the key's range overlaps with the buffer's and decypt * the part that does. */ - if (data_start <= curr_key->end_lsn && curr_key->start_lsn <= data_end) + if (data_start < curr_key->end_lsn && data_end > curr_key->start_lsn) { - dec_off = XLogSegmentOffset(Max(data_start, curr_key->start_lsn), segSize); - dec_end = XLogSegmentOffset(Min(data_end, curr_key->end_lsn), segSize); + char iv_prefix[16]; + off_t dec_off = XLogSegmentOffset(Max(data_start, curr_key->start_lsn), segSize); + off_t dec_end = XLogSegmentOffset(Min(data_end, curr_key->end_lsn), segSize); + size_t dec_sz; + + CalcXLogPageIVPrefix(tli, segno, curr_key->key->base_iv, iv_prefix); /* We have reached the end of the segment */ if (dec_end == 0) @@ -351,28 +356,52 @@ tdeheap_xlog_seg_read(int fd, void *buf, size_t count, off_t offset, } } } - - curr_key = curr_key->next; } return readsz; } -/* IV: TLI(uint32) + XLogRecPtr(uint64)*/ -static inline void -SetXLogPageIVPrefix(TimeLineID tli, XLogRecPtr lsn, char *iv_prefix) +union u128cast { - iv_prefix[0] = (tli >> 24); - iv_prefix[1] = ((tli >> 16) & 0xFF); - iv_prefix[2] = ((tli >> 8) & 0xFF); - iv_prefix[3] = (tli & 0xFF); - - iv_prefix[4] = (lsn >> 56); - iv_prefix[5] = ((lsn >> 48) & 0xFF); - iv_prefix[6] = ((lsn >> 40) & 0xFF); - iv_prefix[7] = ((lsn >> 32) & 0xFF); - iv_prefix[8] = ((lsn >> 24) & 0xFF); - iv_prefix[9] = ((lsn >> 16) & 0xFF); - iv_prefix[10] = ((lsn >> 8) & 0xFF); - iv_prefix[11] = (lsn & 0xFF); + char a[16]; + unsigned __int128 i; +}; + +/* + * Calculate the start IV for an XLog segmenet. + * + * IV: (TLI(uint32) + XLogRecPtr(uint64)) + BaseIV(uint8[12]) + * + * TODO: Make the calculation more like OpenSSL's CTR withot any gaps and + * preferrably without zeroing the lowest bytes for the base IV. + * + * TODO: This code vectorizes poorly in both gcc and clang. + */ +static void +CalcXLogPageIVPrefix(TimeLineID tli, XLogRecPtr lsn, const unsigned char *base_iv, char *iv_prefix) +{ + union u128cast base; + union u128cast iv; + unsigned __int128 offset; + + for (int i = 0; i < 16; i++) +#ifdef WORDS_BIGENDIAN + base.a[i] = base_iv[i]; +#else + base.a[i] = base_iv[15 - i]; +#endif + + /* We do not support wrapping addition in Aes128EncryptedZeroBlocks() */ + base.i &= ~(((unsigned __int128) 1) << 32); + + offset = (((unsigned __int128) tli) << 112) | (((unsigned __int128) lsn) << 32); + + iv.i = base.i + offset; + + for (int i = 0; i < 16; i++) +#ifdef WORDS_BIGENDIAN + iv_prefix[i] = iv.a[i]; +#else + iv_prefix[i] = iv.a[15 - i]; +#endif } diff --git a/contrib/pg_tde/src/catalog/tde_keyring.c b/contrib/pg_tde/src/catalog/tde_keyring.c index 7fd6649ee7358..7637662de5943 100644 --- a/contrib/pg_tde/src/catalog/tde_keyring.c +++ b/contrib/pg_tde/src/catalog/tde_keyring.c @@ -44,11 +44,10 @@ typedef enum ProviderScanType { PROVIDER_SCAN_BY_NAME, PROVIDER_SCAN_BY_ID, - PROVIDER_SCAN_BY_TYPE, PROVIDER_SCAN_ALL } ProviderScanType; -#define PG_TDE_KEYRING_FILENAME "pg_tde_%d_keyring" +#define PG_TDE_KEYRING_FILENAME "%d_providers" #define FILE_KEYRING_TYPE "file" #define VAULTV2_KEYRING_TYPE "vault-v2" @@ -59,13 +58,12 @@ static GenericKeyring *load_keyring_provider_options(ProviderType provider_type, static VaultV2Keyring *load_vaultV2_keyring_provider_options(char *keyring_options); static KmipKeyring *load_kmip_keyring_provider_options(char *keyring_options); static void debug_print_kerying(GenericKeyring *keyring); -static GenericKeyring *load_keyring_provider_from_record(KeyringProvideRecord *provider); +static GenericKeyring *load_keyring_provider_from_record(KeyringProviderRecord *provider); static inline void get_keyring_infofile_path(char *resPath, Oid dbOid); -static bool fetch_next_key_provider(int fd, off_t *curr_pos, KeyringProvideRecord *provider); +static int open_keyring_infofile(Oid dbOid, int flags); +static bool fetch_next_key_provider(int fd, off_t *curr_pos, KeyringProviderRecord *provider); -static uint32 write_key_provider_info(KeyringProvideRecord *provider, - Oid database_id, off_t position, - bool error_if_exists, bool write_xlog); +static void write_key_provider_info(KeyringProviderRecordInFile *record, bool write_xlog); #ifdef FRONTEND @@ -76,25 +74,14 @@ static void simple_list_free(SimplePtrList *list); static List *scan_key_provider_file(ProviderScanType scanType, void *scanKey, Oid dbOid); -PG_FUNCTION_INFO_V1(pg_tde_add_key_provider); -Datum pg_tde_add_key_provider(PG_FUNCTION_ARGS); - +PG_FUNCTION_INFO_V1(pg_tde_add_database_key_provider); PG_FUNCTION_INFO_V1(pg_tde_add_global_key_provider); -Datum pg_tde_add_global_key_provider(PG_FUNCTION_ARGS); - -PG_FUNCTION_INFO_V1(pg_tde_change_key_provider); -Datum pg_tde_change_key_provider(PG_FUNCTION_ARGS); - +PG_FUNCTION_INFO_V1(pg_tde_change_database_key_provider); PG_FUNCTION_INFO_V1(pg_tde_change_global_key_provider); -Datum pg_tde_change_global_key_provider(PG_FUNCTION_ARGS); - static Datum pg_tde_list_all_key_providers_internal(const char *fname, bool global, PG_FUNCTION_ARGS); -PG_FUNCTION_INFO_V1(pg_tde_list_all_key_providers); -Datum pg_tde_list_all_key_providers(PG_FUNCTION_ARGS); - +PG_FUNCTION_INFO_V1(pg_tde_list_all_database_key_providers); PG_FUNCTION_INFO_V1(pg_tde_list_all_global_key_providers); -Datum pg_tde_list_all_global_key_providers(PG_FUNCTION_ARGS); static Datum pg_tde_change_key_provider_internal(PG_FUNCTION_ARGS, Oid dbOid); @@ -149,7 +136,7 @@ tde_provider_info_lock(void) void InitializeKeyProviderInfo(void) { - ereport(LOG, (errmsg("initializing TDE key provider info"))); + ereport(LOG, errmsg("initializing TDE key provider info")); RegisterShmemRequest(&key_provider_info_shmem_routine); on_ext_install(key_provider_startup_cleanup, NULL); } @@ -160,7 +147,7 @@ key_provider_startup_cleanup(int tde_tbl_count, XLogExtensionInstall *ext_info, if (tde_tbl_count > 0) { ereport(WARNING, - (errmsg("failed to perform initialization. database already has %d TDE tables", tde_tbl_count))); + errmsg("failed to perform initialization. database already has %d TDE tables", tde_tbl_count)); return; } cleanup_key_provider_info(ext_info->database_id); @@ -178,9 +165,8 @@ get_keyring_provider_typename(ProviderType p_type) case KMIP_KEY_PROVIDER: return KMIP_KEYRING_TYPE; default: - break; + return NULL; } - return NULL; } static List * @@ -189,10 +175,12 @@ GetAllKeyringProviders(Oid dbOid) return scan_key_provider_file(PROVIDER_SCAN_ALL, NULL, dbOid); } -uint32 -redo_key_provider_info(KeyringProviderXLRecord *xlrec) +void +redo_key_provider_info(KeyringProviderRecordInFile *xlrec) { - return write_key_provider_info(&xlrec->provider, xlrec->database_id, xlrec->offset_in_file, false, false); + LWLockAcquire(tde_provider_info_lock(), LW_EXCLUSIVE); + write_key_provider_info(xlrec, false); + LWLockRelease(tde_provider_info_lock()); } static void @@ -206,7 +194,7 @@ cleanup_key_provider_info(Oid databaseId) } Datum -pg_tde_change_key_provider(PG_FUNCTION_ARGS) +pg_tde_change_database_key_provider(PG_FUNCTION_ARGS) { return pg_tde_change_key_provider_internal(fcinfo, MyDatabaseId); } @@ -214,6 +202,11 @@ pg_tde_change_key_provider(PG_FUNCTION_ARGS) Datum pg_tde_change_global_key_provider(PG_FUNCTION_ARGS) { + if (!superuser()) + ereport(ERROR, + errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to modify global key providers")); + return pg_tde_change_key_provider_internal(fcinfo, GLOBAL_DATA_TDE_OID); } @@ -225,38 +218,39 @@ pg_tde_change_key_provider_internal(PG_FUNCTION_ARGS, Oid dbOid) char *options = text_to_cstring(PG_GETARG_TEXT_PP(2)); int nlen, olen; - KeyringProvideRecord provider; + KeyringProviderRecord provider; /* reports error if not found */ GenericKeyring *keyring = GetKeyProviderByName(provider_name, dbOid); - pfree(keyring); - nlen = strlen(provider_name); if (nlen >= sizeof(provider.provider_name)) ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("too long provider name, maximum lenght is %ld bytes", sizeof(provider.provider_name) - 1))); + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("too long provider name, maximum lenght is %ld bytes", sizeof(provider.provider_name) - 1)); olen = strlen(options); if (olen >= sizeof(provider.options)) ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("too large provider options, maximum size is %ld bytes", sizeof(provider.options) - 1))); + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("too large provider options, maximum size is %ld bytes", sizeof(provider.options) - 1)); /* Struct will be saved to disk so keep clean */ memset(&provider, 0, sizeof(provider)); - provider.provider_id = 0; + provider.provider_id = keyring->keyring_id; memcpy(provider.provider_name, provider_name, nlen); memcpy(provider.options, options, olen); provider.provider_type = get_keyring_provider_from_typename(provider_type); + + pfree(keyring); + modify_key_provider_info(&provider, dbOid, true); PG_RETURN_INT32(provider.provider_id); } Datum -pg_tde_add_key_provider(PG_FUNCTION_ARGS) +pg_tde_add_database_key_provider(PG_FUNCTION_ARGS) { return pg_tde_add_key_provider_internal(fcinfo, MyDatabaseId); } @@ -264,6 +258,11 @@ pg_tde_add_key_provider(PG_FUNCTION_ARGS) Datum pg_tde_add_global_key_provider(PG_FUNCTION_ARGS) { + if (!superuser()) + ereport(ERROR, + errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to modify global key providers")); + return pg_tde_add_key_provider_internal(fcinfo, GLOBAL_DATA_TDE_OID); } @@ -275,19 +274,19 @@ pg_tde_add_key_provider_internal(PG_FUNCTION_ARGS, Oid dbOid) char *options = text_to_cstring(PG_GETARG_TEXT_PP(2)); int nlen, olen; - KeyringProvideRecord provider; + KeyringProviderRecord provider; nlen = strlen(provider_name); if (nlen >= sizeof(provider.provider_name) - 1) ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("too long provider name, maximum lenght is %ld bytes", sizeof(provider.provider_name) - 1))); + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("too long provider name, maximum lenght is %ld bytes", sizeof(provider.provider_name) - 1)); olen = strlen(options); if (olen >= sizeof(provider.options)) ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("too large provider options, maximum size is %ld bytes", sizeof(provider.options) - 1))); + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("too large provider options, maximum size is %ld bytes", sizeof(provider.options) - 1)); /* Struct will be saved to disk so keep clean */ memset(&provider, 0, sizeof(provider)); @@ -301,15 +300,15 @@ pg_tde_add_key_provider_internal(PG_FUNCTION_ARGS, Oid dbOid) } Datum -pg_tde_list_all_key_providers(PG_FUNCTION_ARGS) +pg_tde_list_all_database_key_providers(PG_FUNCTION_ARGS) { - return pg_tde_list_all_key_providers_internal("pg_tde_list_all_key_providers", false, fcinfo); + return pg_tde_list_all_key_providers_internal("pg_tde_list_all_database_key_providers_database", false, fcinfo); } Datum pg_tde_list_all_global_key_providers(PG_FUNCTION_ARGS) { - return pg_tde_list_all_key_providers_internal("pg_tde_list_all_key_providers_global", true, fcinfo); + return pg_tde_list_all_key_providers_internal("pg_tde_list_all_database_key_providers_global", true, fcinfo); } static Datum @@ -327,12 +326,12 @@ pg_tde_list_all_key_providers_internal(const char *fname, bool global, PG_FUNCTI /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("%s: set-valued function called in context that cannot accept a set", fname))); + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("%s: set-valued function called in context that cannot accept a set", fname)); if (!(rsinfo->allowedModes & SFRM_Materialize)) ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("%s: materialize mode required, but it is not allowed in this context", fname))); + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("%s: materialize mode required, but it is not allowed in this context", fname)); /* Switch into long-lived context to construct returned data structures */ per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; @@ -385,189 +384,248 @@ GetKeyProviderByID(int provider_id, Oid dbOid) #endif /* !FRONTEND */ -static uint32 -write_key_provider_info(KeyringProvideRecord *provider, Oid database_id, - off_t position, bool error_if_exists, bool write_xlog) +static void +write_key_provider_info(KeyringProviderRecordInFile *record, bool write_xlog) { off_t bytes_written = 0; - off_t curr_pos = 0; int fd; - int seek_pos = -1; - - /* Named max, but global key provider oids are stored as negative numbers! */ - int max_provider_id = 0; char kp_info_path[MAXPGPATH] = {0}; - KeyringProvideRecord existing_provider; - GenericKeyring *record; - Assert(provider != NULL); + Assert(record != NULL); + Assert(record->offset_in_file >= 0); + Assert(LWLockHeldByMeInMode(tde_provider_info_lock(), LW_EXCLUSIVE)); - if (error_if_exists && provider->provider_id != 0) + get_keyring_infofile_path(kp_info_path, record->database_id); + fd = BasicOpenFile(kp_info_path, O_CREAT | O_RDWR | PG_BINARY); + if (fd < 0) { ereport(ERROR, - (errcode(ERRCODE_DATA_EXCEPTION), errmsg("Invalid write provider call"))); + errcode_for_file_access(), + errmsg("could not open tde file \"%s\": %m", kp_info_path)); } - /* Try to parse the JSON data first: if it doesn't work, don't save it! */ - if (provider->provider_type != UNKNOWN_KEY_PROVIDER) + /* + * emit the xlog here. So that we can handle partial file write errors but + * cannot make new WAL entries during recovery. + */ + if (write_xlog) { - record = load_keyring_provider_from_record(provider); - if (record == NULL) - { - ereport(ERROR, - (errcode(ERRCODE_DATA_EXCEPTION), errmsg("Invalid provider options"))); - } - else - { - pfree(record); - } +#ifndef FRONTEND + XLogBeginInsert(); + XLogRegisterData((char *) record, sizeof(KeyringProviderRecordInFile)); + XLogInsert(RM_TDERMGR_ID, XLOG_TDE_WRITE_KEY_PROVIDER); +#else + Assert(0); +#endif } - get_keyring_infofile_path(kp_info_path, database_id); + bytes_written = pg_pwrite(fd, &(record->provider), + sizeof(KeyringProviderRecord), + record->offset_in_file); + if (bytes_written != sizeof(KeyringProviderRecord)) + { + close(fd); + ereport(ERROR, + errcode_for_file_access(), + errmsg("key provider info file \"%s\" can't be written: %m", + kp_info_path)); + } + if (pg_fsync(fd) != 0) + { + close(fd); + ereport(ERROR, + errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", kp_info_path)); + } + close(fd); +} - LWLockAcquire(tde_provider_info_lock(), LW_EXCLUSIVE); +static void +check_provider_record(KeyringProviderRecord *provider_record) +{ + GenericKeyring *provider; - fd = BasicOpenFile(kp_info_path, O_CREAT | O_RDWR | PG_BINARY); - if (fd < 0) + if (provider_record->provider_type == UNKNOWN_KEY_PROVIDER) { ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not open tde file \"%s\": %m", kp_info_path))); + errcode(ERRCODE_DATA_EXCEPTION), + errmsg("Invalid provider type.")); } - if (position == -1) + + /* Validate that the provider record can be properly parsed. */ + provider = load_keyring_provider_from_record(provider_record); + + if (provider == NULL) { - /* - * we also need to verify the name conflict and generate the next - * provider ID - */ - int before_pos = curr_pos; + ereport(ERROR, + errcode(ERRCODE_DATA_EXCEPTION), + errmsg("Invalid provider options.")); + } - while (fetch_next_key_provider(fd, &curr_pos, &existing_provider)) - { - if (provider->provider_id != 0 && existing_provider.provider_id == provider->provider_id) - { - seek_pos = before_pos; - break; - } - if (strlen(existing_provider.provider_name) > 0 && strcmp(existing_provider.provider_name, provider->provider_name) == 0) - { - if (error_if_exists) - { - close(fd); - ereport(ERROR, - (errcode(ERRCODE_DUPLICATE_OBJECT), - errmsg("key provider \"%s\" already exists", provider->provider_name))); - } - else - { - - seek_pos = before_pos; - provider->provider_id = existing_provider.provider_id; - break; - } - } - if (max_provider_id < abs(existing_provider.provider_id)) - max_provider_id = abs(existing_provider.provider_id); + KeyringValidate(provider); - before_pos = curr_pos; - } - if (seek_pos == -1) - { - provider->provider_id = max_provider_id + 1; + pfree(provider); +} - if (database_id == GLOBAL_DATA_TDE_OID) - { - provider->provider_id = -provider->provider_id; - } - curr_pos = lseek(fd, 0, SEEK_END); - } - else +/* Returns true if the record is found, false otherwise. */ +static bool +get_keyring_info_file_record_by_name(char *provider_name, Oid database_id, + KeyringProviderRecordInFile *record) +{ + off_t current_file_offset = 0; + off_t next_file_offset = 0; + int fd; + KeyringProviderRecord existing_provider; + + Assert(provider_name != NULL); + Assert(record != NULL); + + fd = open_keyring_infofile(database_id, O_RDONLY); + + while (fetch_next_key_provider(fd, &next_file_offset, &existing_provider)) + { + /* Ignore deleted provider records */ + if (existing_provider.provider_type != UNKNOWN_KEY_PROVIDER + && strcmp(existing_provider.provider_name, provider_name) == 0) { - curr_pos = lseek(fd, seek_pos, SEEK_CUR); + record->database_id = database_id; + record->offset_in_file = current_file_offset; + record->provider = existing_provider; + close(fd); + return true; } + current_file_offset = next_file_offset; + } + + /* No matching key provider found */ + close(fd); + return false; +} +/* + * Save the key provider info to the file + */ +void +save_new_key_provider_info(KeyringProviderRecord *provider, Oid databaseId, bool write_xlog) +{ + off_t next_file_offset; + int fd; + KeyringProviderRecord existing_provider; + int max_provider_id = 0; + int new_provider_id; + KeyringProviderRecordInFile file_record; + + Assert(provider != NULL); + + check_provider_record(provider); + + LWLockAcquire(tde_provider_info_lock(), LW_EXCLUSIVE); + + /* + * Validate that the provider name does not collide with an existing + * provider, find the largest existing provider_id and also find the end + * of file offset for appending the provider record. + */ + fd = open_keyring_infofile(databaseId, O_CREAT | O_RDONLY); + + next_file_offset = 0; + while (fetch_next_key_provider(fd, &next_file_offset, &existing_provider)) + { /* - * emit the xlog here. So that we can handle partial file write errors - * but cannot make new WAL entries during recovery. + * abs() is used here because provider_id is negative for global + * providers. */ - if (write_xlog) - { -#ifndef FRONTEND - KeyringProviderXLRecord xlrec; + max_provider_id = Max(max_provider_id, abs(existing_provider.provider_id)); - xlrec.database_id = database_id; - xlrec.offset_in_file = curr_pos; - xlrec.provider = *provider; + /* Ignore deleted records */ + if (existing_provider.provider_type == UNKNOWN_KEY_PROVIDER) + continue; - XLogBeginInsert(); - XLogRegisterData((char *) &xlrec, sizeof(KeyringProviderXLRecord)); - XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_KEY_PROVIDER_KEY); -#else - Assert(0); -#endif + if (strcmp(existing_provider.provider_name, provider->provider_name) == 0) + { + close(fd); + ereport(ERROR, + errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("Key provider \"%s\" already exists.", provider->provider_name)); } } - else + close(fd); + + if (max_provider_id == PG_INT32_MAX) { - /* - * we are performing redo, just go to the position received from the - * xlog and write the record there. No need to verify the name - * conflict and generate the provider ID - */ - curr_pos = lseek(fd, position, SEEK_SET); + ereport(ERROR, + errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg("cannot create key provider, id out of range")); } + new_provider_id = max_provider_id + 1; + provider->provider_id = (databaseId == GLOBAL_DATA_TDE_OID ? -new_provider_id : new_provider_id); - /* - * All good, Just add a new provider - */ - bytes_written = pg_pwrite(fd, provider, sizeof(KeyringProvideRecord), curr_pos); - if (bytes_written != sizeof(KeyringProvideRecord)) + file_record.database_id = databaseId; + file_record.offset_in_file = next_file_offset; + file_record.provider = *provider; + + write_key_provider_info(&file_record, true); + + LWLockRelease(tde_provider_info_lock()); +} + +void +modify_key_provider_info(KeyringProviderRecord *provider, Oid databaseId, bool write_xlog) +{ + KeyringProviderRecordInFile record; + + Assert(provider != NULL); + + check_provider_record(provider); + + LWLockAcquire(tde_provider_info_lock(), LW_EXCLUSIVE); + + if (get_keyring_info_file_record_by_name(provider->provider_name, databaseId, &record) == false) { - close(fd); ereport(ERROR, - (errcode_for_file_access(), - errmsg("key provider info file \"%s\" can't be written: %m", - kp_info_path))); + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("keyring \"%s\" does not exist", provider->provider_name)); } - if (pg_fsync(fd) != 0) + + if (provider->provider_id != record.provider.provider_id) { - close(fd); + /* This should never happen. */ ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not fsync file \"%s\": %m", - kp_info_path))); + errcode(ERRCODE_DATA_EXCEPTION), + errmsg("provider id mismatch %d is not %d", provider->provider_id, record.provider.provider_id)); } - close(fd); - LWLockRelease(tde_provider_info_lock()); - return provider->provider_id; -} + record.provider = *provider; + write_key_provider_info(&record, write_xlog); -/* - * Save the key provider info to the file - */ -uint32 -save_new_key_provider_info(KeyringProvideRecord *provider, Oid databaseId, bool write_xlog) -{ - return write_key_provider_info(provider, databaseId, -1, true, write_xlog); + LWLockRelease(tde_provider_info_lock()); } -uint32 -modify_key_provider_info(KeyringProvideRecord *provider, Oid databaseId, bool write_xlog) +void +delete_key_provider_info(char *provider_name, Oid databaseId, bool write_xlog) { - return write_key_provider_info(provider, databaseId, -1, false, write_xlog); -} + int provider_id; + KeyringProviderRecordInFile record; -uint32 -delete_key_provider_info(int provider_id, Oid databaseId, bool write_xlog) -{ - KeyringProvideRecord kpr; + Assert(provider_name != NULL); + + LWLockAcquire(tde_provider_info_lock(), LW_EXCLUSIVE); + + if (get_keyring_info_file_record_by_name(provider_name, databaseId, &record) == false) + { + ereport(ERROR, + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("key provider \"%s\" does not exist", provider_name)); + } - memset(&kpr, 0, sizeof(KeyringProvideRecord)); - kpr.provider_id = provider_id; + /* Preserve provider_id for deleted records in the file. */ + provider_id = record.provider.provider_id; + memset(&(record.provider), 0, sizeof(KeyringProviderRecord)); + record.provider.provider_id = provider_id; + write_key_provider_info(&record, write_xlog); - return modify_key_provider_info(&kpr, databaseId, write_xlog); + LWLockRelease(tde_provider_info_lock()); } #ifdef FRONTEND @@ -616,7 +674,7 @@ scan_key_provider_file(ProviderScanType scanType, void *scanKey, Oid dbOid) off_t curr_pos = 0; int fd; char kp_info_path[MAXPGPATH] = {0}; - KeyringProvideRecord provider; + KeyringProviderRecord provider; #ifndef FRONTEND List *providers_list = NIL; #else @@ -635,8 +693,8 @@ scan_key_provider_file(ProviderScanType scanType, void *scanKey, Oid dbOid) { LWLockRelease(tde_provider_info_lock()); ereport(DEBUG2, - (errcode_for_file_access(), - errmsg("could not open tde file \"%s\": %m", kp_info_path))); + errcode_for_file_access(), + errmsg("could not open tde file \"%s\": %m", kp_info_path)); return providers_list; } while (fetch_next_key_provider(fd, &curr_pos, &provider)) @@ -650,7 +708,7 @@ scan_key_provider_file(ProviderScanType scanType, void *scanKey, Oid dbOid) } ereport(DEBUG2, - (errmsg("read key provider ID=%d %s", provider.provider_id, provider.provider_name))); + errmsg("read key provider ID=%d %s", provider.provider_id, provider.provider_name)); if (scanType == PROVIDER_SCAN_BY_NAME) { @@ -662,11 +720,6 @@ scan_key_provider_file(ProviderScanType scanType, void *scanKey, Oid dbOid) if (provider.provider_id == *(int *) scanKey) match = true; } - else if (scanType == PROVIDER_SCAN_BY_TYPE) - { - if (provider.provider_type == *(ProviderType *) scanKey) - match = true; - } else if (scanType == PROVIDER_SCAN_ALL) match = true; @@ -692,7 +745,7 @@ scan_key_provider_file(ProviderScanType scanType, void *scanKey, Oid dbOid) } static GenericKeyring * -load_keyring_provider_from_record(KeyringProvideRecord *provider) +load_keyring_provider_from_record(KeyringProviderRecord *provider) { GenericKeyring *keyring = NULL; @@ -716,17 +769,13 @@ load_keyring_provider_options(ProviderType provider_type, char *keyring_options) { case FILE_KEY_PROVIDER: return (GenericKeyring *) load_file_keyring_provider_options(keyring_options); - break; case VAULT_V2_KEY_PROVIDER: return (GenericKeyring *) load_vaultV2_keyring_provider_options(keyring_options); - break; case KMIP_KEY_PROVIDER: return (GenericKeyring *) load_kmip_keyring_provider_options(keyring_options); - break; default: - break; + return NULL; } - return NULL; } static FileKeyring * @@ -745,8 +794,8 @@ load_file_keyring_provider_options(char *keyring_options) if (file_keyring->file_name == NULL || file_keyring->file_name[0] == '\0') { ereport(WARNING, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("file path is missing in the keyring options"))); + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("file path is missing in the keyring options")); return NULL; } @@ -771,11 +820,11 @@ load_vaultV2_keyring_provider_options(char *keyring_options) vaultV2_keyring->vault_mount_path == NULL || vaultV2_keyring->vault_mount_path[0] == '\0') { ereport(WARNING, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("missing in the keyring options:%s%s%s", - (vaultV2_keyring->vault_token != NULL && vaultV2_keyring->vault_token[0] != '\0') ? "" : " token", - (vaultV2_keyring->vault_url != NULL && vaultV2_keyring->vault_url[0] != '\0') ? "" : " url", - (vaultV2_keyring->vault_mount_path != NULL && vaultV2_keyring->vault_mount_path[0] != '\0') ? "" : " mountPath"))); + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("missing in the keyring options:%s%s%s", + (vaultV2_keyring->vault_token != NULL && vaultV2_keyring->vault_token[0] != '\0') ? "" : " token", + (vaultV2_keyring->vault_url != NULL && vaultV2_keyring->vault_url[0] != '\0') ? "" : " url", + (vaultV2_keyring->vault_mount_path != NULL && vaultV2_keyring->vault_mount_path[0] != '\0') ? "" : " mountPath")); return NULL; } @@ -801,12 +850,12 @@ load_kmip_keyring_provider_options(char *keyring_options) strlen(kmip_keyring->kmip_cert_path) == 0) { ereport(WARNING, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("missing in the keyring options:%s%s%s%s", - (kmip_keyring->kmip_host != NULL && kmip_keyring->kmip_host[0] != '\0') ? "" : " host", - (kmip_keyring->kmip_port != NULL && kmip_keyring->kmip_port[0] != '\0') ? "" : " port", - (kmip_keyring->kmip_ca_path != NULL && kmip_keyring->kmip_ca_path[0] != '\0') ? "" : " caPath", - (kmip_keyring->kmip_cert_path != NULL && kmip_keyring->kmip_cert_path[0] != '\0') ? "" : " certPath"))); + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("missing in the keyring options:%s%s%s%s", + (kmip_keyring->kmip_host != NULL && kmip_keyring->kmip_host[0] != '\0') ? "" : " host", + (kmip_keyring->kmip_port != NULL && kmip_keyring->kmip_port[0] != '\0') ? "" : " port", + (kmip_keyring->kmip_ca_path != NULL && kmip_keyring->kmip_ca_path[0] != '\0') ? "" : " caPath", + (kmip_keyring->kmip_cert_path != NULL && kmip_keyring->kmip_cert_path[0] != '\0') ? "" : " certPath")); return NULL; } @@ -850,30 +899,47 @@ get_keyring_infofile_path(char *resPath, Oid dbOid) join_path_components(resPath, pg_tde_get_tde_data_dir(), psprintf(PG_TDE_KEYRING_FILENAME, dbOid)); } +static int +open_keyring_infofile(Oid database_id, int flags) +{ + int fd; + char kp_info_path[MAXPGPATH] = {0}; + + get_keyring_infofile_path(kp_info_path, database_id); + fd = BasicOpenFile(kp_info_path, flags | PG_BINARY); + if (fd < 0) + { + ereport(ERROR, + errcode_for_file_access(), + errmsg("could not open tde file \"%s\": %m", kp_info_path)); + } + return fd; +} + /* * Fetch the next key provider from the file and update the curr_pos */ static bool -fetch_next_key_provider(int fd, off_t *curr_pos, KeyringProvideRecord *provider) +fetch_next_key_provider(int fd, off_t *curr_pos, KeyringProviderRecord *provider) { off_t bytes_read = 0; Assert(provider != NULL); Assert(fd >= 0); - bytes_read = pg_pread(fd, provider, sizeof(KeyringProvideRecord), *curr_pos); + bytes_read = pg_pread(fd, provider, sizeof(KeyringProviderRecord), *curr_pos); *curr_pos += bytes_read; if (bytes_read == 0) return false; - if (bytes_read != sizeof(KeyringProvideRecord)) + if (bytes_read != sizeof(KeyringProviderRecord)) { close(fd); /* Corrupt file */ ereport(ERROR, - (errcode_for_file_access(), - errmsg("key provider info file is corrupted: %m"), - errdetail("invalid key provider record size %ld expected %lu", bytes_read, sizeof(KeyringProvideRecord)))); + errcode_for_file_access(), + errmsg("key provider info file is corrupted: %m"), + errdetail("invalid key provider record size %ld expected %lu", bytes_read, sizeof(KeyringProviderRecord))); } return true; } @@ -881,27 +947,24 @@ fetch_next_key_provider(int fd, off_t *curr_pos, KeyringProvideRecord *provider) ProviderType get_keyring_provider_from_typename(char *provider_type) { - if (provider_type == NULL) - return UNKNOWN_KEY_PROVIDER; - if (strcmp(FILE_KEYRING_TYPE, provider_type) == 0) return FILE_KEY_PROVIDER; - if (strcmp(VAULTV2_KEYRING_TYPE, provider_type) == 0) + else if (strcmp(VAULTV2_KEYRING_TYPE, provider_type) == 0) return VAULT_V2_KEY_PROVIDER; - if (strcmp(KMIP_KEYRING_TYPE, provider_type) == 0) + else if (strcmp(KMIP_KEYRING_TYPE, provider_type) == 0) return KMIP_KEY_PROVIDER; - return UNKNOWN_KEY_PROVIDER; + else + return UNKNOWN_KEY_PROVIDER; } GenericKeyring * GetKeyProviderByName(const char *provider_name, Oid dbOid) { GenericKeyring *keyring = NULL; - #ifndef FRONTEND - static List *providers; + List *providers; #else - static SimplePtrList *providers; + SimplePtrList *providers; #endif providers = scan_key_provider_file(PROVIDER_SCAN_BY_NAME, (void *) provider_name, dbOid); @@ -919,9 +982,9 @@ GetKeyProviderByName(const char *provider_name, Oid dbOid) else { ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("key provider \"%s\" does not exists", provider_name), - errhint("Use pg_tde_add_key_provider interface to create the key provider"))); + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("key provider \"%s\" does not exists", provider_name), + errhint("Create the key provider")); } return keyring; } diff --git a/contrib/pg_tde/src/catalog/tde_keyring_parse_opts.c b/contrib/pg_tde/src/catalog/tde_keyring_parse_opts.c index 6512f260776cc..e811040d82976 100644 --- a/contrib/pg_tde/src/catalog/tde_keyring_parse_opts.c +++ b/contrib/pg_tde/src/catalog/tde_keyring_parse_opts.c @@ -86,8 +86,8 @@ static const char *JK_FIELD_NAMES[JK_FIELDS_TOTAL] = { [JK_FIELD_PATH] = "path", /* - * These values should match pg_tde_add_key_provider_vault_v2 and - * pg_tde_add_key_provider_file SQL interfaces + * These values should match pg_tde_add_database_key_provider_vault_v2 and + * pg_tde_add_database_key_provider_file SQL interfaces */ [JF_FILE_PATH] = "path", [JK_VAULT_TOKEN] = "token", @@ -189,8 +189,8 @@ ParseKeyringJSONOptions(ProviderType provider_type, void *out_opts, char *in_buf if (jerr != JSON_SUCCESS) { ereport(ERROR, - (errmsg("parsing of keyring options failed: %s", - json_errdetail(jerr, jlex)))); + errmsg("parsing of keyring options failed: %s", + json_errdetail(jerr, jlex))); } #if PG_VERSION_NUM >= 170000 diff --git a/contrib/pg_tde/src/catalog/tde_principal_key.c b/contrib/pg_tde/src/catalog/tde_principal_key.c index 3e7db9a9afe88..377bdbdf20e11 100644 --- a/contrib/pg_tde/src/catalog/tde_principal_key.c +++ b/contrib/pg_tde/src/catalog/tde_principal_key.c @@ -49,11 +49,12 @@ #ifndef FRONTEND -PG_FUNCTION_INFO_V1(pg_tde_delete_key_provider); +PG_FUNCTION_INFO_V1(pg_tde_delete_database_key_provider); PG_FUNCTION_INFO_V1(pg_tde_delete_global_key_provider); -PG_FUNCTION_INFO_V1(pg_tde_verify_principal_key); -PG_FUNCTION_INFO_V1(pg_tde_verify_global_principal_key); +PG_FUNCTION_INFO_V1(pg_tde_verify_key); +PG_FUNCTION_INFO_V1(pg_tde_verify_server_key); +PG_FUNCTION_INFO_V1(pg_tde_verify_default_key); typedef struct TdePrincipalKeySharedState { @@ -108,29 +109,14 @@ static void set_principal_key_with_keyring(const char *key_name, static bool pg_tde_is_provider_used(Oid databaseOid, Oid providerId); static bool pg_tde_verify_principal_key_internal(Oid databaseOid); -static Datum pg_tde_delete_key_provider_internal(PG_FUNCTION_ARGS, int is_global); +static Datum pg_tde_delete_key_provider_internal(PG_FUNCTION_ARGS, Oid db_oid); -PG_FUNCTION_INFO_V1(pg_tde_set_default_principal_key); -Datum pg_tde_set_default_principal_key(PG_FUNCTION_ARGS); +PG_FUNCTION_INFO_V1(pg_tde_set_default_key_using_global_key_provider); +PG_FUNCTION_INFO_V1(pg_tde_set_key_using_database_key_provider); +PG_FUNCTION_INFO_V1(pg_tde_set_key_using_global_key_provider); +PG_FUNCTION_INFO_V1(pg_tde_set_server_key_using_global_key_provider); -PG_FUNCTION_INFO_V1(pg_tde_set_principal_key); -Datum pg_tde_set_principal_key(PG_FUNCTION_ARGS); - -PG_FUNCTION_INFO_V1(pg_tde_set_global_principal_key); -Datum pg_tde_set_principal_key(PG_FUNCTION_ARGS); - -PG_FUNCTION_INFO_V1(pg_tde_set_server_principal_key); -Datum pg_tde_set_principal_key(PG_FUNCTION_ARGS); - -enum global_status -{ - GS_LOCAL, - GS_GLOBAL, - GS_SERVER, - GS_DEFAULT -}; - -static void pg_tde_set_principal_key_internal(char *principal_key_name, enum global_status global, char *provider_name, bool ensure_new_key); +static void pg_tde_set_principal_key_internal(Oid providerOid, Oid dbOid, const char *principal_key_name, const char *provider_name, bool ensure_new_key); static const TDEShmemSetupRoutine principal_key_info_shmem_routine = { .init_shared_state = initialize_shared_state, @@ -142,7 +128,7 @@ static const TDEShmemSetupRoutine principal_key_info_shmem_routine = { void InitializePrincipalKeyInfo(void) { - ereport(LOG, (errmsg("Initializing TDE principal key info"))); + ereport(LOG, errmsg("Initializing TDE principal key info")); RegisterShmemRequest(&principal_key_info_shmem_routine); on_ext_install(principal_key_startup_cleanup, NULL); } @@ -188,7 +174,7 @@ initialize_shared_state(void *start_address) { TdePrincipalKeySharedState *sharedState = (TdePrincipalKeySharedState *) start_address; - ereport(LOG, (errmsg("initializing shared state for principal key"))); + ereport(LOG, errmsg("initializing shared state for principal key")); principalKeyLocalState.dsa = NULL; principalKeyLocalState.sharedHash = NULL; @@ -204,7 +190,7 @@ initialize_objects_in_dsa_area(dsa_area *dsa, void *raw_dsa_area) dshash_table *dsh; TdePrincipalKeySharedState *sharedState = principalKeyLocalState.sharedPrincipalKeyState; - ereport(LOG, (errmsg("initializing dsa area objects for principal key"))); + ereport(LOG, errmsg("initializing dsa area objects for principal key")); Assert(sharedState != NULL); @@ -271,7 +257,7 @@ set_principal_key_with_keyring(const char *key_name, const char *provider_name, if (AllowInheritGlobalProviders == false && providerOid != dbOid) { ereport(ERROR, - (errmsg("Usage of global key providers is disabled. Enable it with pg_tde.inherit_global_providers = ON"))); + errmsg("Usage of global key providers is disabled. Enable it with pg_tde.inherit_global_providers = ON")); } /* @@ -285,7 +271,7 @@ set_principal_key_with_keyring(const char *key_name, const char *provider_name, if (provider_name == NULL && !already_has_key) { ereport(ERROR, - (errmsg("provider_name is a required parameter when creating the first principal key for a database"))); + errmsg("provider_name is a required parameter when creating the first principal key for a database")); } if (provider_name != NULL) @@ -306,29 +292,28 @@ set_principal_key_with_keyring(const char *key_name, const char *provider_name, if (kr_ret != KEYRING_CODE_SUCCESS && kr_ret != KEYRING_CODE_RESOURCE_NOT_AVAILABLE) { ereport(ERROR, - (errmsg("failed to retrieve principal key from keyring provider :\"%s\"", new_keyring->provider_name), - errdetail("Error code: %d", kr_ret))); + errmsg("failed to retrieve principal key from keyring provider :\"%s\"", new_keyring->provider_name), + errdetail("Error code: %d", kr_ret)); } } if (keyInfo != NULL && ensure_new_key) { ereport(ERROR, - (errmsg("failed to create principal key: already exists"))); + errmsg("failed to create principal key: already exists")); } if (strlen(key_name) >= sizeof(keyInfo->name)) ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("too long principal key name, maximum lenght is %ld bytes", sizeof(keyInfo->name) - 1))); + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("too long principal key name, maximum lenght is %ld bytes", sizeof(keyInfo->name) - 1)); if (keyInfo == NULL) keyInfo = KeyringGenerateNewKeyAndStore(new_keyring, key_name, PRINCIPAL_KEY_LEN); if (keyInfo == NULL) { - ereport(ERROR, - (errmsg("failed to retrieve/create principal key."))); + ereport(ERROR, errmsg("failed to retrieve/create principal key.")); } new_principal_key = palloc_object(TDEPrincipalKey); @@ -343,19 +328,13 @@ set_principal_key_with_keyring(const char *key_name, const char *provider_name, if (!already_has_key) { /* First key created for the database */ - pg_tde_save_principal_key(&new_principal_key->keyInfo); - - /* XLog the new key */ - XLogBeginInsert(); - XLogRegisterData((char *) &new_principal_key->keyInfo, sizeof(TDEPrincipalKeyInfo)); - XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_PRINCIPAL_KEY); - + pg_tde_save_principal_key(new_principal_key, true); push_principal_key_to_cache(new_principal_key); } else { /* key rotation */ - pg_tde_perform_rotate_key(curr_principal_key, new_principal_key); + pg_tde_perform_rotate_key(curr_principal_key, new_principal_key, true); if (!TDEisInGlobalSpace(curr_principal_key->keyInfo.databaseId)) { @@ -367,6 +346,7 @@ set_principal_key_with_keyring(const char *key_name, const char *provider_name, LWLockRelease(lock_files); pfree(new_keyring); + pfree(new_principal_key); } /* @@ -375,12 +355,59 @@ set_principal_key_with_keyring(const char *key_name, const char *provider_name, void xl_tde_perform_rotate_key(XLogPrincipalKeyRotate *xlrec) { + TDEPrincipalKey *curr_principal_key; + TDEPrincipalKey *new_principal_key; + GenericKeyring *new_keyring; + KeyInfo *keyInfo; + KeyringReturnCodes kr_ret; + LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); - pg_tde_write_map_keydata_file(xlrec->file_size, xlrec->buff); - clear_principal_key_cache(xlrec->databaseId); + curr_principal_key = GetPrincipalKeyNoDefault(xlrec->databaseId, LW_EXCLUSIVE); + + /* Should not happen */ + if (curr_principal_key == NULL) + { + ereport(ERROR, errmsg("failed to retrieve current principal key for database %u.", xlrec->databaseId)); + } + + new_keyring = GetKeyProviderByID(xlrec->keyringId, xlrec->databaseId); + keyInfo = KeyringGetKey(new_keyring, xlrec->keyName, &kr_ret); + + if (kr_ret != KEYRING_CODE_SUCCESS && kr_ret != KEYRING_CODE_RESOURCE_NOT_AVAILABLE) + { + ereport(ERROR, + errmsg("failed to retrieve principal key from keyring provider: \"%s\"", new_keyring->provider_name), + errdetail("Error code: %d", kr_ret)); + } + + /* The new key should be on keyring by this time */ + if (keyInfo == NULL) + { + ereport(ERROR, errmsg("failed to retrieve principal key from keyring for database %u.", xlrec->databaseId)); + } + + new_principal_key = palloc_object(TDEPrincipalKey); + new_principal_key->keyInfo.databaseId = xlrec->databaseId; + new_principal_key->keyInfo.keyringId = new_keyring->keyring_id; + memcpy(new_principal_key->keyInfo.name, keyInfo->name, TDE_KEY_NAME_LEN); + gettimeofday(&new_principal_key->keyInfo.creationTime, NULL); + new_principal_key->keyLength = keyInfo->data.len; + + memcpy(new_principal_key->keyData, keyInfo->data.data, keyInfo->data.len); + + pg_tde_perform_rotate_key(curr_principal_key, new_principal_key, false); + + if (!TDEisInGlobalSpace(curr_principal_key->keyInfo.databaseId)) + { + clear_principal_key_cache(curr_principal_key->keyInfo.databaseId); + push_principal_key_to_cache(new_principal_key); + } LWLockRelease(tde_lwlock_enc_keys()); + + pfree(new_keyring); + pfree(new_principal_key); } /* @@ -456,7 +483,7 @@ principal_key_startup_cleanup(int tde_tbl_count, XLogExtensionInstall *ext_info, if (tde_tbl_count > 0) { ereport(WARNING, - (errmsg("Failed to perform initialization. database already has %d TDE tables", tde_tbl_count))); + errmsg("Failed to perform initialization. database already has %d TDE tables", tde_tbl_count)); return; } @@ -485,81 +512,72 @@ clear_principal_key_cache(Oid databaseId) */ Datum -pg_tde_set_default_principal_key(PG_FUNCTION_ARGS) +pg_tde_set_default_key_using_global_key_provider(PG_FUNCTION_ARGS) { char *principal_key_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); char *provider_name = PG_ARGISNULL(1) ? NULL : text_to_cstring(PG_GETARG_TEXT_PP(1)); bool ensure_new_key = PG_GETARG_BOOL(2); - pg_tde_set_principal_key_internal(principal_key_name, GS_DEFAULT, provider_name, ensure_new_key); + /* Using a global provider for the default encryption setting */ + pg_tde_set_principal_key_internal(GLOBAL_DATA_TDE_OID, DEFAULT_DATA_TDE_OID, principal_key_name, provider_name, ensure_new_key); PG_RETURN_VOID(); } Datum -pg_tde_set_principal_key(PG_FUNCTION_ARGS) +pg_tde_set_key_using_database_key_provider(PG_FUNCTION_ARGS) { char *principal_key_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); char *provider_name = PG_ARGISNULL(1) ? NULL : text_to_cstring(PG_GETARG_TEXT_PP(1)); bool ensure_new_key = PG_GETARG_BOOL(2); - pg_tde_set_principal_key_internal(principal_key_name, GS_LOCAL, provider_name, ensure_new_key); + /* Using a local provider for the current database */ + pg_tde_set_principal_key_internal(MyDatabaseId, MyDatabaseId, principal_key_name, provider_name, ensure_new_key); PG_RETURN_VOID(); } Datum -pg_tde_set_global_principal_key(PG_FUNCTION_ARGS) +pg_tde_set_key_using_global_key_provider(PG_FUNCTION_ARGS) { char *principal_key_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); char *provider_name = PG_ARGISNULL(1) ? NULL : text_to_cstring(PG_GETARG_TEXT_PP(1)); bool ensure_new_key = PG_GETARG_BOOL(2); - pg_tde_set_principal_key_internal(principal_key_name, GS_GLOBAL, provider_name, ensure_new_key); + /* Using a global provider for the current database */ + pg_tde_set_principal_key_internal(GLOBAL_DATA_TDE_OID, MyDatabaseId, principal_key_name, provider_name, ensure_new_key); PG_RETURN_VOID(); } Datum -pg_tde_set_server_principal_key(PG_FUNCTION_ARGS) +pg_tde_set_server_key_using_global_key_provider(PG_FUNCTION_ARGS) { char *principal_key_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); char *provider_name = PG_ARGISNULL(1) ? NULL : text_to_cstring(PG_GETARG_TEXT_PP(1)); bool ensure_new_key = PG_GETARG_BOOL(2); - pg_tde_set_principal_key_internal(principal_key_name, GS_SERVER, provider_name, ensure_new_key); + /* Using a global provider for the global (wal) database */ + pg_tde_set_principal_key_internal(GLOBAL_DATA_TDE_OID, GLOBAL_DATA_TDE_OID, principal_key_name, provider_name, ensure_new_key); PG_RETURN_VOID(); } static void -pg_tde_set_principal_key_internal(char *principal_key_name, enum global_status global, char *provider_name, bool ensure_new_key) +pg_tde_set_principal_key_internal(Oid providerOid, Oid dbOid, const char *key_name, const char *provider_name, bool ensure_new_key) { - Oid providerOid = MyDatabaseId; - Oid dbOid = MyDatabaseId; TDEPrincipalKey *existingDefaultKey = NULL; TDEPrincipalKey existingKeyCopy; - ereport(LOG, (errmsg("Setting principal key [%s : %s] for the database", principal_key_name, provider_name))); + if (providerOid == GLOBAL_DATA_TDE_OID && !superuser()) + ereport(ERROR, + errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to access global key providers")); - if (global == GS_GLOBAL) /* using a global provider for the current - * database */ - { - providerOid = GLOBAL_DATA_TDE_OID; - } - if (global == GS_SERVER) /* using a globla provider for the global - * (wal) database */ - { - providerOid = GLOBAL_DATA_TDE_OID; - dbOid = GLOBAL_DATA_TDE_OID; - } + ereport(LOG, errmsg("Setting principal key [%s : %s] for the database", key_name, provider_name)); - if (global == GS_DEFAULT) /* using a globla provider for the default - * encryption setting */ + if (dbOid == DEFAULT_DATA_TDE_OID) { - providerOid = GLOBAL_DATA_TDE_OID; - dbOid = DEFAULT_DATA_TDE_OID; - /* Do we already have a default key? If yes, look up the name of it */ LWLockAcquire(tde_lwlock_enc_keys(), LW_SHARED); existingDefaultKey = GetPrincipalKeyNoDefault(dbOid, LW_SHARED); @@ -570,13 +588,13 @@ pg_tde_set_principal_key_internal(char *principal_key_name, enum global_status g LWLockRelease(tde_lwlock_enc_keys()); } - set_principal_key_with_keyring(principal_key_name, + set_principal_key_with_keyring(key_name, provider_name, providerOid, dbOid, ensure_new_key); - if (global == GS_DEFAULT && existingDefaultKey != NULL) + if (dbOid == DEFAULT_DATA_TDE_OID && existingDefaultKey != NULL) { /* * In the previous step, we marked a new default provider Now we have @@ -600,32 +618,45 @@ pg_tde_set_principal_key_internal(char *principal_key_name, enum global_status g } } -PG_FUNCTION_INFO_V1(pg_tde_principal_key_info); +PG_FUNCTION_INFO_V1(pg_tde_key_info); Datum -pg_tde_principal_key_info(PG_FUNCTION_ARGS) +pg_tde_key_info(PG_FUNCTION_ARGS) { return pg_tde_get_key_info(fcinfo, MyDatabaseId); } -PG_FUNCTION_INFO_V1(pg_tde_global_principal_key_info); +PG_FUNCTION_INFO_V1(pg_tde_server_key_info); Datum -pg_tde_global_principal_key_info(PG_FUNCTION_ARGS) +pg_tde_server_key_info(PG_FUNCTION_ARGS) { return pg_tde_get_key_info(fcinfo, GLOBAL_DATA_TDE_OID); } +PG_FUNCTION_INFO_V1(pg_tde_default_key_info); +Datum +pg_tde_default_key_info(PG_FUNCTION_ARGS) +{ + return pg_tde_get_key_info(fcinfo, DEFAULT_DATA_TDE_OID); +} + Datum -pg_tde_verify_principal_key(PG_FUNCTION_ARGS) +pg_tde_verify_key(PG_FUNCTION_ARGS) { return pg_tde_verify_principal_key_internal(MyDatabaseId); } Datum -pg_tde_verify_global_principal_key(PG_FUNCTION_ARGS) +pg_tde_verify_server_key(PG_FUNCTION_ARGS) { return pg_tde_verify_principal_key_internal(GLOBAL_DATA_TDE_OID); } +Datum +pg_tde_verify_default_key(PG_FUNCTION_ARGS) +{ + return pg_tde_verify_principal_key_internal(DEFAULT_DATA_TDE_OID); +} + static Datum pg_tde_get_key_info(PG_FUNCTION_ARGS, Oid dbOid) { @@ -641,16 +672,16 @@ pg_tde_get_key_info(PG_FUNCTION_ARGS, Oid dbOid) /* Build a tuple descriptor for our result type */ if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("function returning record called in context that cannot accept type record"))); + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("function returning record called in context that cannot accept type record")); LWLockAcquire(tde_lwlock_enc_keys(), LW_SHARED); principal_key = GetPrincipalKeyNoDefault(dbOid, LW_SHARED); if (principal_key == NULL) { ereport(ERROR, - (errmsg("Principal key does not exists for the database"), - errhint("Use set_principal_key interface to set the principal key"))); + errmsg("Principal key does not exists for the database"), + errhint("Use set_key interface to set the principal key")); } keyring = GetKeyProviderByID(principal_key->keyInfo.keyringId, principal_key->keyInfo.databaseId); @@ -703,7 +734,7 @@ pg_tde_get_key_info(PG_FUNCTION_ARGS, Oid dbOid) static TDEPrincipalKey * get_principal_key_from_keyring(Oid dbOid) { - TDEPrincipalKeyInfo *principalKeyInfo; + TDESignedPrincipalKeyInfo *principalKeyInfo; GenericKeyring *keyring; KeyInfo *keyInfo; KeyringReturnCodes keyring_ret; @@ -715,28 +746,34 @@ get_principal_key_from_keyring(Oid dbOid) if (principalKeyInfo == NULL) return NULL; - keyring = GetKeyProviderByID(principalKeyInfo->keyringId, dbOid); + keyring = GetKeyProviderByID(principalKeyInfo->data.keyringId, dbOid); if (keyring == NULL) ereport(ERROR, - (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("keyring lookup failed for principal key %s, unknown keyring with ID %d", - principalKeyInfo->name, principalKeyInfo->keyringId))); + errcode(ERRCODE_DATA_CORRUPTED), + errmsg("keyring lookup failed for principal key %s, unknown keyring with ID %d", + principalKeyInfo->data.name, principalKeyInfo->data.keyringId)); - keyInfo = KeyringGetKey(keyring, principalKeyInfo->name, &keyring_ret); + keyInfo = KeyringGetKey(keyring, principalKeyInfo->data.name, &keyring_ret); if (keyInfo == NULL) ereport(ERROR, - (errcode(ERRCODE_NO_DATA_FOUND), - errmsg("failed to retrieve principal key %s from keyring with ID %d", - principalKeyInfo->name, principalKeyInfo->keyringId))); + errcode(ERRCODE_NO_DATA_FOUND), + errmsg("failed to retrieve principal key %s from keyring with ID %d", + principalKeyInfo->data.name, principalKeyInfo->data.keyringId)); principalKey = palloc_object(TDEPrincipalKey); - principalKey->keyInfo = *principalKeyInfo; + principalKey->keyInfo = principalKeyInfo->data; memcpy(principalKey->keyData, keyInfo->data.data, keyInfo->data.len); principalKey->keyLength = keyInfo->data.len; Assert(dbOid == principalKey->keyInfo.databaseId); + if (!pg_tde_verify_principal_key_info(principalKeyInfo, principalKey)) + ereport(ERROR, + errcode(ERRCODE_DATA_CORRUPTED), + errmsg("Failed to verify principal key header for key %s, incorrect principal key or corrupted key file", + principalKeyInfo->data.name)); + pfree(keyInfo); pfree(keyring); pfree(principalKeyInfo); @@ -833,7 +870,7 @@ GetPrincipalKey(Oid dbOid, LWLockMode lockMode) *newPrincipalKey = *principalKey; newPrincipalKey->keyInfo.databaseId = dbOid; - pg_tde_save_principal_key(&newPrincipalKey->keyInfo); + pg_tde_save_principal_key(newPrincipalKey, false); push_principal_key_to_cache(newPrincipalKey); @@ -903,18 +940,12 @@ pg_tde_is_provider_used(Oid databaseOid, Oid providerId) /* We have to verify that it isn't currently used by any database */ - rel = table_open(DatabaseRelationId, AccessShareLock); scan = systable_beginscan(rel, 0, false, NULL, 0, NULL); - while ((tuple = systable_getnext(scan)) != NULL) + while (HeapTupleIsValid(tuple = systable_getnext(scan))) { - if (!HeapTupleIsValid(tuple)) - { - break; - } - dbOid = ((Form_pg_database) GETSTRUCT(tuple))->oid; principal_key = GetPrincipalKeyNoDefault(dbOid, LW_EXCLUSIVE); @@ -924,7 +955,7 @@ pg_tde_is_provider_used(Oid databaseOid, Oid providerId) continue; } - if (providerId == principal_key->keyInfo.keyringId && principal_key->keyInfo.databaseId == GLOBAL_DATA_TDE_OID) + if (providerId == principal_key->keyInfo.keyringId) { systable_endscan(scan); table_close(rel, AccessShareLock); @@ -968,7 +999,7 @@ pg_tde_rotate_default_key_for_database(TDEPrincipalKey *oldKey, TDEPrincipalKey newKey->keyInfo.databaseId = oldKey->keyInfo.databaseId; /* key rotation */ - pg_tde_perform_rotate_key(oldKey, newKey); + pg_tde_perform_rotate_key(oldKey, newKey, true); if (!TDEisInGlobalSpace(newKey->keyInfo.databaseId)) { @@ -996,7 +1027,6 @@ pg_tde_update_global_principal_key_everywhere(TDEPrincipalKey *oldKey, TDEPrinci pg_tde_rotate_default_key_for_database(principal_key, newKey); } - /* * Take row exclusive lock, as we do not want anybody to create/drop a * database in parallel. If it happens, its not the end of the world, but @@ -1006,17 +1036,15 @@ pg_tde_update_global_principal_key_everywhere(TDEPrincipalKey *oldKey, TDEPrinci scan = systable_beginscan(rel, 0, false, NULL, 0, NULL); - while ((tuple = systable_getnext(scan)) != NULL) + while (HeapTupleIsValid(tuple = systable_getnext(scan))) { dbOid = ((Form_pg_database) GETSTRUCT(tuple))->oid; - principal_key = GetPrincipalKeyNoDefault(dbOid, LW_EXCLUSIVE); if (pg_tde_is_same_principal_key(oldKey, principal_key)) { pg_tde_rotate_default_key_for_database(principal_key, newKey); } - } systable_endscan(scan); @@ -1024,30 +1052,33 @@ pg_tde_update_global_principal_key_everywhere(TDEPrincipalKey *oldKey, TDEPrinci } Datum -pg_tde_delete_key_provider(PG_FUNCTION_ARGS) +pg_tde_delete_database_key_provider(PG_FUNCTION_ARGS) { - return pg_tde_delete_key_provider_internal(fcinfo, 0); + return pg_tde_delete_key_provider_internal(fcinfo, MyDatabaseId); } Datum pg_tde_delete_global_key_provider(PG_FUNCTION_ARGS) { - return pg_tde_delete_key_provider_internal(fcinfo, 1); + if (!superuser()) + ereport(ERROR, + errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to modify global key providers")); + + return pg_tde_delete_key_provider_internal(fcinfo, GLOBAL_DATA_TDE_OID); } Datum -pg_tde_delete_key_provider_internal(PG_FUNCTION_ARGS, int is_global) +pg_tde_delete_key_provider_internal(PG_FUNCTION_ARGS, Oid db_oid) { char *provider_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); - Oid db_oid = (is_global == 1) ? GLOBAL_DATA_TDE_OID : MyDatabaseId; GenericKeyring *provider = GetKeyProviderByName(provider_name, db_oid); int provider_id; bool provider_used; if (provider == NULL) { - ereport(ERROR, - (errmsg("Keyring provider not found"))); + ereport(ERROR, errmsg("Keyring provider not found")); } provider_id = provider->keyring_id; @@ -1058,10 +1089,10 @@ pg_tde_delete_key_provider_internal(PG_FUNCTION_ARGS, int is_global) if (provider_used) { ereport(ERROR, - (errmsg("Can't delete a provider which is currently in use"))); + errmsg("Can't delete a provider which is currently in use")); } - delete_key_provider_info(provider_id, db_oid, true); + delete_key_provider_info(provider_name, db_oid, true); PG_RETURN_VOID(); } @@ -1080,13 +1111,13 @@ pg_tde_verify_principal_key_internal(Oid databaseOid) if (fromKeyring == NULL) { ereport(ERROR, - (errmsg("principal key not configured for current database"))); + errmsg("principal key not configured for current database")); } if (fromCache != NULL && (fromKeyring->keyLength != fromCache->keyLength || memcmp(fromKeyring->keyData, fromCache->keyData, fromCache->keyLength) != 0)) { ereport(ERROR, - (errmsg("key returned from keyring and cached in pg_tde differ"))); + errmsg("key returned from keyring and cached in pg_tde differ")); } LWLockRelease(tde_lwlock_enc_keys()); diff --git a/contrib/pg_tde/src/common/pg_tde_shmem.c b/contrib/pg_tde/src/common/pg_tde_shmem.c index 9b511437bbf12..37ad019a5f459 100644 --- a/contrib/pg_tde/src/common/pg_tde_shmem.c +++ b/contrib/pg_tde/src/common/pg_tde_shmem.c @@ -77,7 +77,7 @@ TdeShmemInit(void) LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); /* Create or attach to the shared memory state */ - ereport(NOTICE, (errmsg("TdeShmemInit: requested %ld bytes", required_shmem_size))); + ereport(NOTICE, errmsg("TdeShmemInit: requested %ld bytes", required_shmem_size)); tdeState = ShmemInitStruct("pg_tde", required_shmem_size, &found); if (!found) @@ -110,7 +110,7 @@ TdeShmemInit(void) Assert(dsa_area_size > 0); tdeState->rawDsaArea = p; - ereport(LOG, (errmsg("creating DSA area of size %lu", dsa_area_size))); + ereport(LOG, errmsg("creating DSA area of size %lu", dsa_area_size)); dsa = dsa_create_in_place(tdeState->rawDsaArea, dsa_area_size, LWLockNewTrancheId(), 0); @@ -125,7 +125,7 @@ TdeShmemInit(void) if (routine->init_dsa_area_objects) routine->init_dsa_area_objects(dsa, tdeState->rawDsaArea); } - ereport(LOG, (errmsg("setting no limit to DSA area of size %lu", dsa_area_size))); + ereport(LOG, errmsg("setting no limit to DSA area of size %lu", dsa_area_size)); dsa_set_size_limit(dsa, -1); /* Let it grow outside the shared * memory */ diff --git a/contrib/pg_tde/src/common/pg_tde_utils.c b/contrib/pg_tde/src/common/pg_tde_utils.c index 598c33d2ff6af..5067a8d04816e 100644 --- a/contrib/pg_tde/src/common/pg_tde_utils.c +++ b/contrib/pg_tde/src/common/pg_tde_utils.c @@ -47,8 +47,8 @@ pg_tde_is_encrypted(PG_FUNCTION_ARGS) if (RelFileLocatorBackendIsTemp(rlocator) && !rel->rd_islocaltemp) ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("we cannot check if temporary relations from other backends are encrypted"))); + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("we cannot check if temporary relations from other backends are encrypted")); key = GetSMGRRelationKey(rlocator); diff --git a/contrib/pg_tde/src/encryption/enc_aes.c b/contrib/pg_tde/src/encryption/enc_aes.c index f450c317057f9..cb6f265f7da8f 100644 --- a/contrib/pg_tde/src/encryption/enc_aes.c +++ b/contrib/pg_tde/src/encryption/enc_aes.c @@ -75,14 +75,14 @@ AesRunCtr(EVP_CIPHER_CTX **ctxPtr, int enc, const unsigned char *key, const unsi if (EVP_CipherInit_ex(*ctxPtr, cipher_ctr_ecb, NULL, key, iv, enc) == 0) ereport(ERROR, - (errmsg("EVP_CipherInit_ex failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL)))); + errmsg("EVP_CipherInit_ex failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))); EVP_CIPHER_CTX_set_padding(*ctxPtr, 0); } if (EVP_CipherUpdate(*ctxPtr, out, &out_len, in, in_len) == 0) ereport(ERROR, - (errmsg("EVP_CipherUpdate failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL)))); + errmsg("EVP_CipherUpdate failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))); Assert(out_len == in_len); } @@ -101,17 +101,17 @@ AesRunCbc(int enc, const unsigned char *key, const unsigned char *iv, const unsi if (EVP_CipherInit_ex(ctx, cipher_cbc, NULL, key, iv, enc) == 0) ereport(ERROR, - (errmsg("EVP_CipherInit_ex failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL)))); + errmsg("EVP_CipherInit_ex failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))); EVP_CIPHER_CTX_set_padding(ctx, 0); if (EVP_CipherUpdate(ctx, out, &out_len, in, in_len) == 0) ereport(ERROR, - (errmsg("EVP_CipherUpdate failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL)))); + errmsg("EVP_CipherUpdate failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))); if (EVP_CipherFinal_ex(ctx, out + out_len, &out_len_final) == 0) ereport(ERROR, - (errmsg("EVP_CipherFinal_ex failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL)))); + errmsg("EVP_CipherFinal_ex failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))); /* * We encrypt one block (16 bytes) Our expectation is that the result @@ -150,35 +150,35 @@ AesGcmEncrypt(const unsigned char *key, const unsigned char *iv, const unsigned if (EVP_EncryptInit_ex(ctx, cipher_gcm, NULL, NULL, NULL) == 0) ereport(ERROR, - (errmsg("EVP_EncryptInit_ex failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL)))); + errmsg("EVP_EncryptInit_ex failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))); if (EVP_CIPHER_CTX_set_padding(ctx, 0) == 0) ereport(ERROR, - (errmsg("EVP_CIPHER_CTX_set_padding failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL)))); + errmsg("EVP_CIPHER_CTX_set_padding failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))); if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, 16, NULL) == 0) ereport(ERROR, - (errmsg("EVP_CTRL_GCM_SET_IVLEN failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL)))); + errmsg("EVP_CTRL_GCM_SET_IVLEN failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))); if (EVP_EncryptInit_ex(ctx, NULL, NULL, key, iv) == 0) ereport(ERROR, - (errmsg("EVP_EncryptInit_ex failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL)))); + errmsg("EVP_EncryptInit_ex failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))); if (EVP_EncryptUpdate(ctx, NULL, &out_len, (unsigned char *) aad, aad_len) == 0) ereport(ERROR, - (errmsg("EVP_CipherUpdate failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL)))); + errmsg("EVP_CipherUpdate failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))); if (EVP_EncryptUpdate(ctx, out, &out_len, in, in_len) == 0) ereport(ERROR, - (errmsg("EVP_CipherUpdate failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL)))); + errmsg("EVP_CipherUpdate failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))); if (EVP_EncryptFinal_ex(ctx, out + out_len, &out_len_final) == 0) ereport(ERROR, - (errmsg("EVP_CipherFinal_ex failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL)))); + errmsg("EVP_CipherFinal_ex failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))); if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) == 0) ereport(ERROR, - (errmsg("EVP_CTRL_GCM_GET_TAG failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL)))); + errmsg("EVP_CTRL_GCM_GET_TAG failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))); /* * We encrypt one block (16 bytes) Our expectation is that the result @@ -205,31 +205,31 @@ AesGcmDecrypt(const unsigned char *key, const unsigned char *iv, const unsigned if (EVP_DecryptInit_ex(ctx, cipher_gcm, NULL, NULL, NULL) == 0) ereport(ERROR, - (errmsg("EVP_EncryptInit_ex failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL)))); + errmsg("EVP_EncryptInit_ex failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))); if (EVP_CIPHER_CTX_set_padding(ctx, 0) == 0) ereport(ERROR, - (errmsg("EVP_CIPHER_CTX_set_padding failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL)))); + errmsg("EVP_CIPHER_CTX_set_padding failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))); if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, 16, NULL) == 0) ereport(ERROR, - (errmsg("EVP_CTRL_GCM_SET_IVLEN failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL)))); + errmsg("EVP_CTRL_GCM_SET_IVLEN failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))); if (EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv) == 0) ereport(ERROR, - (errmsg("EVP_EncryptInit_ex failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL)))); + errmsg("EVP_EncryptInit_ex failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))); if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) == 0) ereport(ERROR, - (errmsg("EVP_CTRL_GCM_SET_TAG failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL)))); + errmsg("EVP_CTRL_GCM_SET_TAG failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))); if (EVP_DecryptUpdate(ctx, NULL, &out_len, aad, aad_len) == 0) ereport(ERROR, - (errmsg("EVP_CipherUpdate failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL)))); + errmsg("EVP_CipherUpdate failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))); if (EVP_DecryptUpdate(ctx, out, &out_len, in, in_len) == 0) ereport(ERROR, - (errmsg("EVP_CipherUpdate failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL)))); + errmsg("EVP_CipherUpdate failed. OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))); if (EVP_DecryptFinal_ex(ctx, out + out_len, &out_len_final) == 0) { diff --git a/contrib/pg_tde/src/encryption/enc_tde.c b/contrib/pg_tde/src/encryption/enc_tde.c index f4779037a207d..aafb3adb541b3 100644 --- a/contrib/pg_tde/src/encryption/enc_tde.c +++ b/contrib/pg_tde/src/encryption/enc_tde.c @@ -53,8 +53,8 @@ pg_tde_crypt_simple(const char *iv_prefix, uint32 start_offset, const char *data iv_prefix_debug(iv_prefix, ivp_debug); ereport(LOG, - (errmsg("%s: Start offset: %lu Data_Len: %u, aes_start_block: %lu, aes_end_block: %lu, IV prefix: %s", - context ? context : "", start_offset, data_len, aes_start_block, aes_end_block, ivp_debug))); + errmsg("%s: Start offset: %lu Data_Len: %u, aes_start_block: %lu, aes_end_block: %lu, IV prefix: %s", + context ? context : "", start_offset, data_len, aes_start_block, aes_end_block, ivp_debug)); } #endif @@ -94,8 +94,8 @@ pg_tde_crypt_complex(const char *iv_prefix, uint32 start_offset, const char *dat iv_prefix_debug(iv_prefix, ivp_debug); ereport(LOG, - (errmsg("%s: Batch-No:%d Start offset: %lu Data_Len: %u, batch_start_block: %lu, batch_end_block: %lu, IV prefix: %s", - context ? context : "", batch_no, start_offset, data_len, batch_start_block, batch_end_block, ivp_debug))); + errmsg("%s: Batch-No:%d Start offset: %lu Data_Len: %u, batch_start_block: %lu, batch_end_block: %lu, IV prefix: %s", + context ? context : "", batch_no, start_offset, data_len, batch_start_block, batch_end_block, ivp_debug)); } #endif diff --git a/contrib/pg_tde/src/include/access/pg_tde_tdemap.h b/contrib/pg_tde/src/include/access/pg_tde_tdemap.h index 75e19a955d066..81b11fc399a4a 100644 --- a/contrib/pg_tde/src/include/access/pg_tde_tdemap.h +++ b/contrib/pg_tde/src/include/access/pg_tde_tdemap.h @@ -19,7 +19,9 @@ #define TDE_KEY_TYPE_GLOBAL 0x04 #define TDE_KEY_TYPE_WAL_UNENCRYPTED 0x08 #define TDE_KEY_TYPE_WAL_ENCRYPTED 0x10 -#define MAP_ENTRY_VALID (TDE_KEY_TYPE_SMGR | TDE_KEY_TYPE_GLOBAL) + +#define INTERNAL_KEY_LEN 16 +#define INTERNAL_KEY_IV_LEN 16 typedef struct InternalKey { @@ -28,6 +30,7 @@ typedef struct InternalKey * pg_tde_read/write_one_keydata() */ uint8 key[INTERNAL_KEY_LEN]; + uint8 base_iv[INTERNAL_KEY_IV_LEN]; uint32 rel_type; XLogRecPtr start_lsn; @@ -42,6 +45,13 @@ typedef struct InternalKey #define MAP_ENTRY_EMPTY_IV_SIZE 16 #define MAP_ENTRY_EMPTY_AEAD_TAG_SIZE 16 +typedef struct +{ + TDEPrincipalKeyInfo data; + unsigned char sign_iv[16]; + unsigned char aead_tag[16]; +} TDESignedPrincipalKeyInfo; + /* We do not need the dbOid since the entries are stored in a file per db */ typedef struct TDEMapEntry { @@ -56,8 +66,7 @@ typedef struct TDEMapEntry typedef struct XLogRelKey { - TDEMapEntry mapEntry; - TDEPrincipalKeyInfo pkInfo; + RelFileLocator rlocator; } XLogRelKey; /* @@ -89,11 +98,11 @@ extern WALKeyCacheRec *pg_tde_get_wal_cache_keys(void); extern void pg_tde_wal_last_key_set_lsn(XLogRecPtr lsn, const char *keyfile_path); extern InternalKey *pg_tde_create_smgr_key(const RelFileLocatorBackend *newrlocator); +extern void pg_tde_create_smgr_key_perm_redo(const RelFileLocator *newrlocator); extern void pg_tde_create_wal_key(InternalKey *rel_key_data, const RelFileLocator *newrlocator, uint32 flags); -extern void pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset); -extern void pg_tde_write_key_map_entry_redo(const TDEMapEntry *write_map_entry, TDEPrincipalKeyInfo *principal_key_info); +extern void pg_tde_free_key_map_entry(const RelFileLocator *rlocator); -#define PG_TDE_MAP_FILENAME "pg_tde_%d_map" +#define PG_TDE_MAP_FILENAME "%d_keys" static inline void pg_tde_set_db_file_path(Oid dbOid, char *path) @@ -105,10 +114,11 @@ extern InternalKey *GetSMGRRelationKey(RelFileLocatorBackend rel); extern void pg_tde_delete_tde_files(Oid dbOid); -extern TDEPrincipalKeyInfo *pg_tde_get_principal_key_info(Oid dbOid); -extern void pg_tde_save_principal_key(TDEPrincipalKeyInfo *principal_key_info); -extern void pg_tde_save_principal_key_redo(TDEPrincipalKeyInfo *principal_key_info); -extern void pg_tde_perform_rotate_key(TDEPrincipalKey *principal_key, TDEPrincipalKey *new_principal_key); +extern TDESignedPrincipalKeyInfo *pg_tde_get_principal_key_info(Oid dbOid); +extern bool pg_tde_verify_principal_key_info(TDESignedPrincipalKeyInfo *signed_key_info, const TDEPrincipalKey *principal_key); +extern void pg_tde_save_principal_key(const TDEPrincipalKey *principal_key, bool write_xlog); +extern void pg_tde_save_principal_key_redo(const TDESignedPrincipalKeyInfo *signed_key_info); +extern void pg_tde_perform_rotate_key(TDEPrincipalKey *principal_key, TDEPrincipalKey *new_principal_key, bool write_xlog); extern void pg_tde_write_map_keydata_file(off_t size, char *file_data); const char *tde_sprint_key(InternalKey *k); diff --git a/contrib/pg_tde/src/include/access/pg_tde_xlog.h b/contrib/pg_tde/src/include/access/pg_tde_xlog.h index 22b8bd9116039..21823478d7580 100644 --- a/contrib/pg_tde/src/include/access/pg_tde_xlog.h +++ b/contrib/pg_tde/src/include/access/pg_tde_xlog.h @@ -13,13 +13,12 @@ #include "postgres.h" -/* TDE XLOG resource manager */ +/* TDE XLOG record types */ #define XLOG_TDE_ADD_RELATION_KEY 0x00 #define XLOG_TDE_ADD_PRINCIPAL_KEY 0x10 -#define XLOG_TDE_EXTENSION_INSTALL_KEY 0x20 -#define XLOG_TDE_ROTATE_KEY 0x30 -#define XLOG_TDE_ADD_KEY_PROVIDER_KEY 0x40 -#define XLOG_TDE_FREE_MAP_ENTRY 0x50 +#define XLOG_TDE_ROTATE_PRINCIPAL_KEY 0x20 +#define XLOG_TDE_WRITE_KEY_PROVIDER 0x30 +#define XLOG_TDE_INSTALL_EXTENSION 0x40 /* ID 140 is registered for Percona TDE extension: https://wiki.postgresql.org/wiki/CustomWALResourceManagers */ #define RM_TDERMGR_ID 140 diff --git a/contrib/pg_tde/src/include/access/pg_tde_xlog_encrypt.h b/contrib/pg_tde/src/include/access/pg_tde_xlog_smgr.h similarity index 73% rename from contrib/pg_tde/src/include/access/pg_tde_xlog_encrypt.h rename to contrib/pg_tde/src/include/access/pg_tde_xlog_smgr.h index c501bb637253d..ce8df4b8769e8 100644 --- a/contrib/pg_tde/src/include/access/pg_tde_xlog_encrypt.h +++ b/contrib/pg_tde/src/include/access/pg_tde_xlog_smgr.h @@ -1,13 +1,13 @@ /*------------------------------------------------------------------------- * - * pg_tde_xlog_encrypt.h + * pg_tde_xlog_smgr.h * Encrypted XLog storage manager * *------------------------------------------------------------------------- */ -#ifndef PG_TDE_XLOGENCRYPT_H -#define PG_TDE_XLOGENCRYPT_H +#ifndef PG_TDE_XLOGSMGR_H +#define PG_TDE_XLOGSMGR_H #include "postgres.h" @@ -15,4 +15,4 @@ extern Size TDEXLogEncryptStateSize(void); extern void TDEXLogShmemInit(void); extern void TDEXLogSmgrInit(void); -#endif /* PG_TDE_XLOGENCRYPT_H */ +#endif /* PG_TDE_XLOGSMGR_H */ diff --git a/contrib/pg_tde/src/include/catalog/tde_keyring.h b/contrib/pg_tde/src/include/catalog/tde_keyring.h index f2d3521fc2df1..0cd7e4470e91e 100644 --- a/contrib/pg_tde/src/include/catalog/tde_keyring.h +++ b/contrib/pg_tde/src/include/catalog/tde_keyring.h @@ -14,32 +14,33 @@ #include "keyring/keyring_api.h" /* This record goes into key provider info file */ -typedef struct KeyringProvideRecord +typedef struct KeyringProviderRecord { int provider_id; char provider_name[MAX_PROVIDER_NAME_LEN]; char options[MAX_KEYRING_OPTION_LEN]; ProviderType provider_type; -} KeyringProvideRecord; +} KeyringProviderRecord; -typedef struct KeyringProviderXLRecord +/* This struct also keeps some context of where the record belongs */ +typedef struct KeyringProviderRecordInFile { Oid database_id; off_t offset_in_file; - KeyringProvideRecord provider; -} KeyringProviderXLRecord; + KeyringProviderRecord provider; +} KeyringProviderRecordInFile; extern GenericKeyring *GetKeyProviderByName(const char *provider_name, Oid dbOid); extern GenericKeyring *GetKeyProviderByID(int provider_id, Oid dbOid); extern ProviderType get_keyring_provider_from_typename(char *provider_type); extern void InitializeKeyProviderInfo(void); -extern uint32 save_new_key_provider_info(KeyringProvideRecord *provider, - Oid databaseId, bool write_xlog); -extern uint32 modify_key_provider_info(KeyringProvideRecord *provider, +extern void save_new_key_provider_info(KeyringProviderRecord *provider, Oid databaseId, bool write_xlog); -extern uint32 delete_key_provider_info(int provider_id, - Oid databaseId, bool write_xlog); -extern uint32 redo_key_provider_info(KeyringProviderXLRecord *xlrec); +extern void modify_key_provider_info(KeyringProviderRecord *provider, + Oid databaseId, bool write_xlog); +extern void delete_key_provider_info(char *provider_name, + Oid databaseId, bool write_xlog); +extern void redo_key_provider_info(KeyringProviderRecordInFile *xlrec); extern bool ParseKeyringJSONOptions(ProviderType provider_type, void *out_opts, char *in_buf, int buf_len); diff --git a/contrib/pg_tde/src/include/catalog/tde_principal_key.h b/contrib/pg_tde/src/include/catalog/tde_principal_key.h index 29ea86975ca44..d7ee7806154ef 100644 --- a/contrib/pg_tde/src/include/catalog/tde_principal_key.h +++ b/contrib/pg_tde/src/include/catalog/tde_principal_key.h @@ -37,8 +37,8 @@ typedef struct TDEPrincipalKey typedef struct XLogPrincipalKeyRotate { Oid databaseId; - off_t file_size; - char buff[FLEXIBLE_ARRAY_MEMBER]; + Oid keyringId; + char keyName[PRINCIPAL_KEY_NAME_LEN]; } XLogPrincipalKeyRotate; #define SizeoOfXLogPrincipalKeyRotate offsetof(XLogPrincipalKeyRotate, buff) diff --git a/contrib/pg_tde/src/include/config.h b/contrib/pg_tde/src/include/config.h deleted file mode 100644 index 393cefbfb7dee..0000000000000 --- a/contrib/pg_tde/src/include/config.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef TDE_CONFIG_H -#define TDE_CONFIG_H - -#define PACKAGE_NAME "pg_tde" -#define PACKAGE_VERSION "1.0.0-rc" - -#define PACKAGE_STRING PACKAGE_NAME" "PACKAGE_VERSION - -#define PACKAGE_TARNAME "pg_tde" -#define PACKAGE_BUGREPORT "https://github.com/percona/pg_tde/issues" - -#endif /* TDE_CONFIG_H */ diff --git a/contrib/pg_tde/src/include/keyring/keyring_api.h b/contrib/pg_tde/src/include/keyring/keyring_api.h index 6848c896bda4a..89be8282f8500 100644 --- a/contrib/pg_tde/src/include/keyring/keyring_api.h +++ b/contrib/pg_tde/src/include/keyring/keyring_api.h @@ -61,6 +61,7 @@ typedef struct TDEKeyringRoutine { KeyInfo *(*keyring_get_key) (GenericKeyring *keyring, const char *key_name, KeyringReturnCodes *returnCode); void (*keyring_store_key) (GenericKeyring *keyring, KeyInfo *key); + void (*keyring_validate) (GenericKeyring *keyring); } TDEKeyringRoutine; typedef struct FileKeyring @@ -87,9 +88,10 @@ typedef struct KmipKeyring char *kmip_cert_path; } KmipKeyring; -extern void RegisterKeyProvider(const TDEKeyringRoutine *routine, ProviderType type); +extern void RegisterKeyProviderType(const TDEKeyringRoutine *routine, ProviderType type); extern KeyInfo *KeyringGetKey(GenericKeyring *keyring, const char *key_name, KeyringReturnCodes *returnCode); extern KeyInfo *KeyringGenerateNewKeyAndStore(GenericKeyring *keyring, const char *key_name, unsigned key_len); +extern void KeyringValidate(GenericKeyring *keyring); #endif /* KEYRING_API_H */ diff --git a/contrib/pg_tde/src/include/pg_tde.h b/contrib/pg_tde/src/include/pg_tde.h index 167efea4bfdcf..d3c14387045a0 100644 --- a/contrib/pg_tde/src/include/pg_tde.h +++ b/contrib/pg_tde/src/include/pg_tde.h @@ -8,6 +8,10 @@ #ifndef PG_TDE_H #define PG_TDE_H +#define PG_TDE_NAME "pg_tde" +#define PG_TDE_VERSION "1.0.0-rc" +#define PG_TDE_VERSION_STRING PG_TDE_NAME " " PG_TDE_VERSION + #define PG_TDE_DATA_DIR "pg_tde" typedef struct XLogExtensionInstall diff --git a/contrib/pg_tde/src/include/pg_tde_defs.h b/contrib/pg_tde/src/include/pg_tde_defs.h deleted file mode 100644 index 1402c6a8fbef5..0000000000000 --- a/contrib/pg_tde/src/include/pg_tde_defs.h +++ /dev/null @@ -1,16 +0,0 @@ -/*------------------------------------------------------------------------- - * - * pg_tde_defs.h - * src/include/pg_tde_defs.h - * - *------------------------------------------------------------------------- - */ -#ifndef PG_TDE_DEFS_H -#define PG_TDE_DEFS_H - - -extern const char *pg_tde_package_string(void); -extern const char *pg_tde_package_name(void); -extern const char *pg_tde_package_version(void); - -#endif /* PG_TDE_DEFS_H */ diff --git a/contrib/pg_tde/src/include/transam/pg_tde_xact_handler.h b/contrib/pg_tde/src/include/transam/pg_tde_xact_handler.h deleted file mode 100644 index 7838423ae9502..0000000000000 --- a/contrib/pg_tde/src/include/transam/pg_tde_xact_handler.h +++ /dev/null @@ -1,18 +0,0 @@ -/*------------------------------------------------------------------------- - * - * pg_tde_xact_handler.h - * TDE transaction handling. - * - *------------------------------------------------------------------------- - */ -#ifndef PG_TDE_XACT_HANDLER_H -#define PG_TDE_XACT_HANDLER_H - -#include "postgres.h" -#include "storage/relfilelocator.h" - -extern void RegisterTdeXactCallbacks(void); -extern void RegisterEntryForDeletion(const RelFileLocator *rlocator, off_t map_entry_offset, bool atCommit); - - -#endif /* PG_TDE_XACT_HANDLER_H */ diff --git a/contrib/pg_tde/src/keyring/keyring_api.c b/contrib/pg_tde/src/keyring/keyring_api.c index 15c9e93b2a83b..f6d9785eef756 100644 --- a/contrib/pg_tde/src/keyring/keyring_api.c +++ b/contrib/pg_tde/src/keyring/keyring_api.c @@ -11,31 +11,31 @@ #include #include -typedef struct KeyProviders +typedef struct RegisteredKeyProviderType { TDEKeyringRoutine *routine; ProviderType type; -} KeyProviders; +} RegisteredKeyProviderType; #ifndef FRONTEND -static List *registeredKeyProviders = NIL; +static List *registeredKeyProviderTypes = NIL; #else -static SimplePtrList registeredKeyProviders = {NULL, NULL}; +static SimplePtrList registeredKeyProviderTypes = {NULL, NULL}; #endif -static KeyProviders *find_key_provider(ProviderType type); +static RegisteredKeyProviderType *find_key_provider_type(ProviderType type); static void KeyringStoreKey(GenericKeyring *keyring, KeyInfo *key); static KeyInfo *KeyringGenerateNewKey(const char *key_name, unsigned key_len); #ifndef FRONTEND -static KeyProviders * -find_key_provider(ProviderType type) +static RegisteredKeyProviderType * +find_key_provider_type(ProviderType type) { ListCell *lc; - foreach(lc, registeredKeyProviders) + foreach(lc, registeredKeyProviderTypes) { - KeyProviders *kp = (KeyProviders *) lfirst(lc); + RegisteredKeyProviderType *kp = (RegisteredKeyProviderType *) lfirst(lc); if (kp->type == type) { @@ -45,14 +45,14 @@ find_key_provider(ProviderType type) return NULL; } #else -static KeyProviders * -find_key_provider(ProviderType type) +static RegisteredKeyProviderType * +find_key_provider_type(ProviderType type) { SimplePtrListCell *lc; - for (lc = registeredKeyProviders.head; lc; lc = lc->next) + for (lc = registeredKeyProviderTypes.head; lc; lc = lc->next) { - KeyProviders *kp = (KeyProviders *) lc->ptr; + RegisteredKeyProviderType *kp = (RegisteredKeyProviderType *) lc->ptr; if (kp->type == type) { @@ -64,9 +64,9 @@ find_key_provider(ProviderType type) #endif /* !FRONTEND */ void -RegisterKeyProvider(const TDEKeyringRoutine *routine, ProviderType type) +RegisterKeyProviderType(const TDEKeyringRoutine *routine, ProviderType type) { - KeyProviders *kp; + RegisteredKeyProviderType *kp; #ifndef FRONTEND MemoryContext oldcontext; #endif @@ -74,35 +74,36 @@ RegisterKeyProvider(const TDEKeyringRoutine *routine, ProviderType type) Assert(routine != NULL); Assert(routine->keyring_get_key != NULL); Assert(routine->keyring_store_key != NULL); + Assert(routine->keyring_validate != NULL); - kp = find_key_provider(type); + kp = find_key_provider_type(type); if (kp) ereport(ERROR, - (errmsg("Key provider of type %d already registered", type))); + errmsg("Key provider of type %d already registered", type)); #ifndef FRONTEND oldcontext = MemoryContextSwitchTo(TopMemoryContext); #endif - kp = palloc_object(KeyProviders); + kp = palloc_object(RegisteredKeyProviderType); kp->routine = (TDEKeyringRoutine *) routine; kp->type = type; #ifndef FRONTEND - registeredKeyProviders = lappend(registeredKeyProviders, kp); + registeredKeyProviderTypes = lappend(registeredKeyProviderTypes, kp); MemoryContextSwitchTo(oldcontext); #else - simple_ptr_list_append(®isteredKeyProviders, kp); + simple_ptr_list_append(®isteredKeyProviderTypes, kp); #endif } KeyInfo * KeyringGetKey(GenericKeyring *keyring, const char *key_name, KeyringReturnCodes *returnCode) { - KeyProviders *kp = find_key_provider(keyring->type); + RegisteredKeyProviderType *kp = find_key_provider_type(keyring->type); if (kp == NULL) { ereport(WARNING, - (errmsg("Key provider of type %d not registered", keyring->type))); + errmsg("Key provider of type %d not registered", keyring->type)); *returnCode = KEYRING_CODE_INVALID_PROVIDER; return NULL; } @@ -112,11 +113,11 @@ KeyringGetKey(GenericKeyring *keyring, const char *key_name, KeyringReturnCodes static void KeyringStoreKey(GenericKeyring *keyring, KeyInfo *key) { - KeyProviders *kp = find_key_provider(keyring->type); + RegisteredKeyProviderType *kp = find_key_provider_type(keyring->type); if (kp == NULL) ereport(ERROR, - (errmsg("Key provider of type %d not registered", keyring->type))); + errmsg("Key provider of type %d not registered", keyring->type)); kp->routine->keyring_store_key(keyring, key); } @@ -148,3 +149,15 @@ KeyringGenerateNewKeyAndStore(GenericKeyring *keyring, const char *key_name, uns return key; } + +void +KeyringValidate(GenericKeyring *keyring) +{ + RegisteredKeyProviderType *kp = find_key_provider_type(keyring->type); + + if (kp == NULL) + ereport(ERROR, + errmsg("Key provider of type %d not registered", keyring->type)); + + kp->routine->keyring_validate(keyring); +} diff --git a/contrib/pg_tde/src/keyring/keyring_file.c b/contrib/pg_tde/src/keyring/keyring_file.c index 1abb93a650f64..5d1f83c1349ef 100644 --- a/contrib/pg_tde/src/keyring/keyring_file.c +++ b/contrib/pg_tde/src/keyring/keyring_file.c @@ -28,16 +28,18 @@ static KeyInfo *get_key_by_name(GenericKeyring *keyring, const char *key_name, KeyringReturnCodes *return_code); static void set_key_by_name(GenericKeyring *keyring, KeyInfo *key); +static void validate(GenericKeyring *keyring); const TDEKeyringRoutine keyringFileRoutine = { .keyring_get_key = get_key_by_name, - .keyring_store_key = set_key_by_name + .keyring_store_key = set_key_by_name, + .keyring_validate = validate, }; void InstallFileKeyring(void) { - RegisterKeyProvider(&keyringFileRoutine, FILE_KEY_PROVIDER); + RegisterKeyProviderType(&keyringFileRoutine, FILE_KEY_PROVIDER); } static KeyInfo * @@ -78,10 +80,10 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, KeyringReturnCode /* Corrupt file */ *return_code = KEYRING_CODE_DATA_CORRUPTED; ereport(WARNING, - (errcode_for_file_access(), - errmsg("keyring file \"%s\" is corrupted: %m", - file_keyring->file_name), - errdetail("invalid key size %lu expected %lu", bytes_read, sizeof(KeyInfo)))); + errcode_for_file_access(), + errmsg("keyring file \"%s\" is corrupted: %m", + file_keyring->file_name), + errdetail("invalid key size %lu expected %lu", bytes_read, sizeof(KeyInfo))); return NULL; } if (strncasecmp(key->name, key_name, sizeof(key->name)) == 0) @@ -111,15 +113,15 @@ set_key_by_name(GenericKeyring *keyring, KeyInfo *key) if (existing_key) { ereport(ERROR, - (errmsg("Key with name %s already exists in keyring", key->name))); + errmsg("Key with name %s already exists in keyring", key->name)); } fd = BasicOpenFile(file_keyring->file_name, O_CREAT | O_RDWR | PG_BINARY); if (fd < 0) { ereport(ERROR, - (errcode_for_file_access(), - errmsg("Failed to open keyring file %s :%m", file_keyring->file_name))); + errcode_for_file_access(), + errmsg("Failed to open keyring file %s: %m", file_keyring->file_name)); } /* Write key to the end of file */ curr_pos = lseek(fd, 0, SEEK_END); @@ -128,18 +130,34 @@ set_key_by_name(GenericKeyring *keyring, KeyInfo *key) { close(fd); ereport(ERROR, - (errcode_for_file_access(), - errmsg("keyring file \"%s\" can't be written: %m", - file_keyring->file_name))); + errcode_for_file_access(), + errmsg("keyring file \"%s\" can't be written: %m", + file_keyring->file_name)); } if (pg_fsync(fd) != 0) { close(fd); ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not fsync file \"%s\": %m", - file_keyring->file_name))); + errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", + file_keyring->file_name)); } close(fd); } + +static void +validate(GenericKeyring *keyring) +{ + FileKeyring *file_keyring = (FileKeyring *) keyring; + int fd = BasicOpenFile(file_keyring->file_name, O_CREAT | O_RDWR | PG_BINARY); + + if (fd < 0) + { + ereport(ERROR, + errcode_for_file_access(), + errmsg("Failed to open keyring file %s: %m", file_keyring->file_name)); + } + + close(fd); +} diff --git a/contrib/pg_tde/src/keyring/keyring_kmip.c b/contrib/pg_tde/src/keyring/keyring_kmip.c index fc9a8735cf083..86b438a920886 100644 --- a/contrib/pg_tde/src/keyring/keyring_kmip.c +++ b/contrib/pg_tde/src/keyring/keyring_kmip.c @@ -26,16 +26,18 @@ static void set_key_by_name(GenericKeyring *keyring, KeyInfo *key); static KeyInfo *get_key_by_name(GenericKeyring *keyring, const char *key_name, KeyringReturnCodes *return_code); +static void validate(GenericKeyring *keyring); const TDEKeyringRoutine keyringKmipRoutine = { .keyring_get_key = get_key_by_name, - .keyring_store_key = set_key_by_name + .keyring_store_key = set_key_by_name, + .keyring_validate = validate, }; void InstallKmipKeyring(void) { - RegisterKeyProvider(&keyringKmipRoutine, KMIP_KEY_PROVIDER); + RegisterKeyProviderType(&keyringKmipRoutine, KMIP_KEY_PROVIDER); } typedef struct KmipCtx @@ -55,21 +57,21 @@ kmipSslConnect(KmipCtx *ctx, KmipKeyring *kmip_keyring, bool throw_error) if (SSL_CTX_use_certificate_file(ctx->ssl, kmip_keyring->kmip_cert_path, SSL_FILETYPE_PEM) != 1) { SSL_CTX_free(ctx->ssl); - ereport(level, (errmsg("SSL error: Loading the client certificate failed"))); + ereport(level, errmsg("SSL error: Loading the client certificate failed")); return false; } if (SSL_CTX_use_PrivateKey_file(ctx->ssl, kmip_keyring->kmip_cert_path, SSL_FILETYPE_PEM) != 1) { SSL_CTX_free(ctx->ssl); - ereport(level, (errmsg("SSL error: Loading the client key failed"))); + ereport(level, errmsg("SSL error: Loading the client key failed")); return false; } if (SSL_CTX_load_verify_locations(ctx->ssl, kmip_keyring->kmip_ca_path, NULL) != 1) { SSL_CTX_free(ctx->ssl); - ereport(level, (errmsg("SSL error: Loading the CA certificate failed"))); + ereport(level, errmsg("SSL error: Loading the CA certificate failed")); return false; } @@ -77,7 +79,7 @@ kmipSslConnect(KmipCtx *ctx, KmipKeyring *kmip_keyring, bool throw_error) if (ctx->bio == NULL) { SSL_CTX_free(ctx->ssl); - ereport(level, (errmsg("SSL error: BIO_new_ssl_connect failed"))); + ereport(level, errmsg("SSL error: BIO_new_ssl_connect failed")); return false; } @@ -89,7 +91,7 @@ kmipSslConnect(KmipCtx *ctx, KmipKeyring *kmip_keyring, bool throw_error) { BIO_free_all(ctx->bio); SSL_CTX_free(ctx->ssl); - ereport(level, (errmsg("SSL error: BIO_do_connect failed"))); + ereport(level, errmsg("SSL error: BIO_do_connect failed")); return false; } @@ -113,7 +115,7 @@ set_key_by_name(GenericKeyring *keyring, KeyInfo *key) SSL_CTX_free(ctx.ssl); if (result != 0) - ereport(ERROR, (errmsg("KMIP server reported error on register symmetric key: %i", result))); + ereport(ERROR, errmsg("KMIP server reported error on register symmetric key: %i", result)); } static KeyInfo * @@ -156,7 +158,7 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, KeyringReturnCode if (ids_found > 1) { - ereport(WARNING, (errmsg("KMIP server contains multiple results for key, ignoring"))); + ereport(WARNING, errmsg("KMIP server contains multiple results for key, ignoring")); *return_code = KEYRING_CODE_RESOURCE_NOT_AVAILABLE; BIO_free_all(ctx.bio); SSL_CTX_free(ctx.ssl); @@ -174,7 +176,7 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, KeyringReturnCode if (result != 0) { - ereport(WARNING, (errmsg("KMIP server LOCATEd key, but GET failed with %i", result))); + ereport(WARNING, errmsg("KMIP server LOCATEd key, but GET failed with %i", result)); *return_code = KEYRING_CODE_RESOURCE_NOT_AVAILABLE; pfree(key); BIO_free_all(ctx.bio); @@ -184,7 +186,7 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, KeyringReturnCode if (key->data.len > sizeof(key->data.data)) { - ereport(WARNING, (errmsg("keyring provider returned invalid key size: %d", key->data.len))); + ereport(WARNING, errmsg("keyring provider returned invalid key size: %d", key->data.len)); *return_code = KEYRING_CODE_INVALID_KEY_SIZE; pfree(key); BIO_free_all(ctx.bio); @@ -204,3 +206,15 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, KeyringReturnCode return key; } + +static void +validate(GenericKeyring *keyring) +{ + KmipKeyring *kmip_keyring = (KmipKeyring *) keyring; + KmipCtx ctx; + + kmipSslConnect(&ctx, kmip_keyring, true); + + BIO_free_all(ctx.bio); + SSL_CTX_free(ctx.ssl); +} diff --git a/contrib/pg_tde/src/keyring/keyring_vault.c b/contrib/pg_tde/src/keyring/keyring_vault.c index c2d41323d512a..03a924237195a 100644 --- a/contrib/pg_tde/src/keyring/keyring_vault.c +++ b/contrib/pg_tde/src/keyring/keyring_vault.c @@ -70,16 +70,18 @@ static bool curl_perform(VaultV2Keyring *keyring, const char *url, CurlString *o static void set_key_by_name(GenericKeyring *keyring, KeyInfo *key); static KeyInfo *get_key_by_name(GenericKeyring *keyring, const char *key_name, KeyringReturnCodes *return_code); +static void validate(GenericKeyring *keyring); const TDEKeyringRoutine keyringVaultV2Routine = { .keyring_get_key = get_key_by_name, - .keyring_store_key = set_key_by_name + .keyring_store_key = set_key_by_name, + .keyring_validate = validate, }; void InstallVaultV2Keyring(void) { - RegisterKeyProvider(&keyringVaultV2Routine, VAULT_V2_KEY_PROVIDER); + RegisterKeyProviderType(&keyringVaultV2Routine, VAULT_V2_KEY_PROVIDER); } static bool @@ -197,8 +199,8 @@ set_key_by_name(GenericKeyring *keyring, KeyInfo *key) if (!curl_perform(vault_keyring, url, &str, &httpCode, jsonText)) { ereport(ERROR, - (errmsg("HTTP(S) request to keyring provider \"%s\" failed", - vault_keyring->keyring.provider_name))); + errmsg("HTTP(S) request to keyring provider \"%s\" failed", + vault_keyring->keyring.provider_name)); } if (str.ptr != NULL) @@ -206,8 +208,8 @@ set_key_by_name(GenericKeyring *keyring, KeyInfo *key) if (httpCode / 100 != 2) ereport(ERROR, - (errmsg("Invalid HTTP response from keyring provider \"%s\": %ld", - vault_keyring->keyring.provider_name, httpCode))); + errmsg("Invalid HTTP response from keyring provider \"%s\": %ld", + vault_keyring->keyring.provider_name, httpCode)); } static KeyInfo * @@ -232,8 +234,8 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, KeyringReturnCode { *return_code = KEYRING_CODE_INVALID_KEY_SIZE; ereport(WARNING, - (errmsg("HTTP(S) request to keyring provider \"%s\" failed", - vault_keyring->keyring.provider_name))); + errmsg("HTTP(S) request to keyring provider \"%s\" failed", + vault_keyring->keyring.provider_name)); goto cleanup; } @@ -247,8 +249,8 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, KeyringReturnCode { *return_code = KEYRING_CODE_INVALID_RESPONSE; ereport(WARNING, - (errmsg("HTTP(S) request to keyring provider \"%s\" returned invalid response %li", - vault_keyring->keyring.provider_name, httpCode))); + errmsg("HTTP(S) request to keyring provider \"%s\" returned invalid response %li", + vault_keyring->keyring.provider_name, httpCode)); goto cleanup; } @@ -263,8 +265,8 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, KeyringReturnCode { *return_code = KEYRING_CODE_INVALID_RESPONSE; ereport(WARNING, - (errmsg("HTTP(S) request to keyring provider \"%s\" returned incorrect JSON: %s", - vault_keyring->keyring.provider_name, json_errdetail(json_error, jlex)))); + errmsg("HTTP(S) request to keyring provider \"%s\" returned incorrect JSON: %s", + vault_keyring->keyring.provider_name, json_errdetail(json_error, jlex))); goto cleanup; } @@ -283,8 +285,8 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, KeyringReturnCode { *return_code = KEYRING_CODE_INVALID_KEY_SIZE; ereport(WARNING, - (errmsg("keyring provider \"%s\" returned invalid key size: %d", - vault_keyring->keyring.provider_name, key->data.len))); + errmsg("keyring provider \"%s\" returned invalid key size: %d", + vault_keyring->keyring.provider_name, key->data.len)); pfree(key); key = NULL; goto cleanup; @@ -300,6 +302,41 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, KeyringReturnCode return key; } +static void +validate(GenericKeyring *keyring) +{ + VaultV2Keyring *vault_keyring = (VaultV2Keyring *) keyring; + char url[VAULT_URL_MAX_LEN]; + CurlString str; + long httpCode = 0; + + /* + * Validate connection by listing available keys at the root level of the + * mount point + */ + snprintf(url, VAULT_URL_MAX_LEN, "%s/v1/%s/metadata/?list=true", + vault_keyring->vault_url, vault_keyring->vault_mount_path); + + if (!curl_perform(vault_keyring, url, &str, &httpCode, NULL)) + { + ereport(ERROR, + errmsg("HTTP(S) request to keyring provider \"%s\" failed", + vault_keyring->keyring.provider_name)); + } + + /* If the mount point doesn't have any secrets yet, we'll get a 404. */ + if (httpCode != 200 && httpCode != 404) + { + ereport(ERROR, + errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("Listing secrets of \"%s\" at mountpoint \"%s\" failed", + vault_keyring->vault_url, vault_keyring->vault_mount_path)); + } + + if (str.ptr != NULL) + pfree(str.ptr); +} + /* * JSON parser routines * diff --git a/contrib/pg_tde/src/pg_tde.c b/contrib/pg_tde/src/pg_tde.c index a6db0811449ca..8576c41c7d923 100644 --- a/contrib/pg_tde/src/pg_tde.c +++ b/contrib/pg_tde/src/pg_tde.c @@ -13,13 +13,12 @@ #include "postgres.h" #include "funcapi.h" #include "pg_tde.h" -#include "transam/pg_tde_xact_handler.h" #include "miscadmin.h" #include "storage/ipc.h" #include "storage/lwlock.h" #include "storage/shmem.h" #include "access/pg_tde_xlog.h" -#include "access/pg_tde_xlog_encrypt.h" +#include "access/pg_tde_xlog_smgr.h" #include "encryption/enc_aes.h" #include "access/pg_tde_tdemap.h" #include "access/xlog.h" @@ -32,7 +31,6 @@ #include "keyring/keyring_vault.h" #include "keyring/keyring_kmip.h" #include "utils/builtins.h" -#include "pg_tde_defs.h" #include "smgr/pg_tde_smgr.h" #include "catalog/tde_global_space.h" #include "utils/percona.h" @@ -79,7 +77,7 @@ tde_shmem_request(void) prev_shmem_request_hook(); RequestAddinShmemSpace(sz); RequestNamedLWLockTranche(TDE_TRANCHE_NAME, required_locks); - ereport(LOG, (errmsg("tde_shmem_request: requested %ld bytes", sz))); + ereport(LOG, errmsg("tde_shmem_request: requested %ld bytes", sz)); } static void @@ -122,7 +120,6 @@ _PG_init(void) prev_shmem_startup_hook = shmem_startup_hook; shmem_startup_hook = tde_shmem_startup; - RegisterTdeXactCallbacks(); InstallFileKeyring(); InstallVaultV2Keyring(); InstallKmipKeyring(); @@ -148,7 +145,7 @@ pg_tde_extension_initialize(PG_FUNCTION_ARGS) */ XLogBeginInsert(); XLogRegisterData((char *) &xlrec, sizeof(XLogExtensionInstall)); - XLogInsert(RM_TDERMGR_ID, XLOG_TDE_EXTENSION_INSTALL_KEY); + XLogInsert(RM_TDERMGR_ID, XLOG_TDE_INSTALL_EXTENSION); PG_RETURN_NULL(); } @@ -171,8 +168,8 @@ on_ext_install(pg_tde_on_ext_install_callback function, void *arg) { if (on_ext_install_index >= MAX_ON_INSTALLS) ereport(FATAL, - (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), - errmsg_internal("out of on extension install slots"))); + errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg_internal("out of on extension install slots")); on_ext_install_list[on_ext_install_index].function = function; on_ext_install_list[on_ext_install_index].arg = arg; @@ -190,9 +187,9 @@ pg_tde_init_data_dir(void) { if (MakePGDirectory(PG_TDE_DATA_DIR) < 0) ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not create tde directory \"%s\": %m", - PG_TDE_DATA_DIR))); + errcode_for_file_access(), + errmsg("could not create tde directory \"%s\": %m", + PG_TDE_DATA_DIR)); } } @@ -222,7 +219,7 @@ run_extension_install_callbacks(XLogExtensionInstall *xlrec, bool redo) Datum pg_tde_version(PG_FUNCTION_ARGS) { - PG_RETURN_TEXT_P(cstring_to_text(pg_tde_package_string())); + PG_RETURN_TEXT_P(cstring_to_text(PG_TDE_VERSION_STRING)); } Datum diff --git a/contrib/pg_tde/src/pg_tde_change_key_provider.c b/contrib/pg_tde/src/pg_tde_change_key_provider.c index 0663c381f0a8f..87db2c5d819b9 100644 --- a/contrib/pg_tde/src/pg_tde_change_key_provider.c +++ b/contrib/pg_tde/src/pg_tde_change_key_provider.c @@ -116,7 +116,7 @@ main(int argc, char *argv[]) char *cptr = tdedir; bool provider_found = false; GenericKeyring *keyring = NULL; - KeyringProvideRecord provider; + KeyringProviderRecord provider; Oid db_oid; diff --git a/contrib/pg_tde/src/pg_tde_defs.c b/contrib/pg_tde/src/pg_tde_defs.c deleted file mode 100644 index 22d2602cbe56a..0000000000000 --- a/contrib/pg_tde/src/pg_tde_defs.c +++ /dev/null @@ -1,36 +0,0 @@ -/*------------------------------------------------------------------------- - * - * pg_tde_defs.c - * The configure script generates config.h which contains the package_* defs - * and these defines conflicts with the PG defines. - * This file is used to provide the package version string to the extension - * without including the config.h file. - * - * IDENTIFICATION - * contrib/pg_tde/src/pg_tde_defs.c - * - *------------------------------------------------------------------------- - */ - - -#include "config.h" -#include "pg_tde_defs.h" - - -/* Returns package version */ -const char * -pg_tde_package_string(void) -{ - return PACKAGE_STRING; -} - -const char * -pg_tde_package_name(void) -{ - return PACKAGE_NAME; -} -const char * -pg_tde_package_version(void) -{ - return PACKAGE_VERSION; -} diff --git a/contrib/pg_tde/src/pg_tde_event_capture.c b/contrib/pg_tde/src/pg_tde_event_capture.c index 9683d4a56ea97..d6b7a7ab4bb0b 100644 --- a/contrib/pg_tde/src/pg_tde_event_capture.c +++ b/contrib/pg_tde/src/pg_tde_event_capture.c @@ -16,6 +16,7 @@ #include "utils/builtins.h" #include "catalog/pg_class.h" #include "commands/defrem.h" +#include "commands/sequence.h" #include "access/table.h" #include "access/relation.h" #include "catalog/pg_event_trigger.h" @@ -28,11 +29,9 @@ #include "miscadmin.h" #include "access/tableam.h" #include "catalog/tde_global_space.h" -#include "executor/spi.h" /* Global variable that gets set at ddl start and cleard out at ddl end*/ -static TdeCreateEvent tdeCurrentCreateEvent = {.tid = {.value = 0},.relation = NULL}; -static bool alterSetAccessMethod = false; +static TdeCreateEvent tdeCurrentCreateEvent = {.tid = {.value = 0}}; static void reset_current_tde_create_event(void); static Oid get_tde_table_am_oid(void); @@ -46,33 +45,35 @@ GetCurrentTdeCreateEvent(void) return &tdeCurrentCreateEvent; } +static bool +shouldEncryptTable(const char *accessMethod) +{ + if (accessMethod) + return strcmp(accessMethod, "tde_heap") == 0; + else + return strcmp(default_table_access_method, "tde_heap") == 0; +} + static void -checkEncryptionClause(const char *accessMethod) +checkPrincipalKeyConfigured(void) { - if (accessMethod && strcmp(accessMethod, "tde_heap") == 0) - { - tdeCurrentCreateEvent.encryptMode = true; - } - else if ((accessMethod == NULL || accessMethod[0] == 0) && strcmp(default_table_access_method, "tde_heap") == 0) - { - tdeCurrentCreateEvent.encryptMode = true; - } + if (!pg_tde_principal_key_configured(MyDatabaseId)) + ereport(ERROR, + errmsg("principal key not configured"), + errhint("create one using pg_tde_set_key before using encrypted tables")); +} +static void +checkEncryptionStatus(void) +{ if (tdeCurrentCreateEvent.encryptMode) { - if (!pg_tde_principal_key_configured(MyDatabaseId)) - { - ereport(ERROR, - (errmsg("principal key not configured"), - errhint("create one using pg_tde_set_principal_key before using encrypted tables"))); - - } + checkPrincipalKeyConfigured(); } - - if (EnforceEncryption && !tdeCurrentCreateEvent.encryptMode) + else if (EnforceEncryption) { ereport(ERROR, - (errmsg("pg_tde.enforce_encryption is ON, only the tde_heap access method is allowed."))); + errmsg("pg_tde.enforce_encryption is ON, only the tde_heap access method is allowed.")); } } @@ -84,9 +85,8 @@ validateCurrentEventTriggerState(bool mightStartTransaction) if (RecoveryInProgress()) { reset_current_tde_create_event(); - return; } - if (tdeCurrentCreateEvent.tid.value != InvalidFullTransactionId.value && tid.value != tdeCurrentCreateEvent.tid.value) + else if (tdeCurrentCreateEvent.tid.value != InvalidFullTransactionId.value && tid.value != tdeCurrentCreateEvent.tid.value) { /* There was a failed query, end event trigger didn't execute */ reset_current_tde_create_event(); @@ -114,110 +114,101 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) /* Ensure this function is being called as an event trigger */ if (!CALLED_AS_EVENT_TRIGGER(fcinfo)) /* internal error */ ereport(ERROR, - (errmsg("Function can only be fired by event trigger manager"))); + errmsg("Function can only be fired by event trigger manager")); - trigdata = (EventTriggerData *) fcinfo->context; + trigdata = castNode(EventTriggerData, fcinfo->context); parsetree = trigdata->parsetree; if (IsA(parsetree, IndexStmt)) { - IndexStmt *stmt = (IndexStmt *) parsetree; - Oid relationId = RangeVarGetRelid(stmt->relation, NoLock, true); + IndexStmt *stmt = castNode(IndexStmt, parsetree); + Relation rel; validateCurrentEventTriggerState(true); tdeCurrentCreateEvent.tid = GetCurrentFullTransactionId(); - tdeCurrentCreateEvent.baseTableOid = relationId; - tdeCurrentCreateEvent.relation = stmt->relation; - - if (relationId != InvalidOid) - { - LOCKMODE lockmode = AccessShareLock; /* TODO. Verify lock mode? */ - Relation rel = table_open(relationId, lockmode); + rel = table_openrv(stmt->relation, AccessShareLock); - if (rel->rd_rel->relam == get_tde_table_am_oid()) - { - /* We are creating the index on encrypted table */ - /* set the global state */ - tdeCurrentCreateEvent.encryptMode = true; - } - table_close(rel, lockmode); + tdeCurrentCreateEvent.baseTableOid = rel->rd_id; - if (tdeCurrentCreateEvent.encryptMode) - { - checkEncryptionClause(""); - } + if (rel->rd_rel->relam == get_tde_table_am_oid()) + { + /* + * We are creating an index on an encrypted table so set the + * global state. + */ + tdeCurrentCreateEvent.encryptMode = true; } - else - ereport(DEBUG1, (errmsg("Failed to get relation Oid for relation:%s", stmt->relation->relname))); + /* Hold on to lock until end of transaction */ + table_close(rel, NoLock); + + if (tdeCurrentCreateEvent.encryptMode) + checkPrincipalKeyConfigured(); } else if (IsA(parsetree, CreateStmt)) { - CreateStmt *stmt = (CreateStmt *) parsetree; - const char *accessMethod = stmt->accessMethod; + CreateStmt *stmt = castNode(CreateStmt, parsetree); validateCurrentEventTriggerState(true); tdeCurrentCreateEvent.tid = GetCurrentFullTransactionId(); + if (shouldEncryptTable(stmt->accessMethod)) + tdeCurrentCreateEvent.encryptMode = true; - tdeCurrentCreateEvent.relation = stmt->relation; - - checkEncryptionClause(accessMethod); + checkEncryptionStatus(); } else if (IsA(parsetree, CreateTableAsStmt)) { - CreateTableAsStmt *stmt = (CreateTableAsStmt *) parsetree; - const char *accessMethod = stmt->into->accessMethod; + CreateTableAsStmt *stmt = castNode(CreateTableAsStmt, parsetree); validateCurrentEventTriggerState(true); tdeCurrentCreateEvent.tid = GetCurrentFullTransactionId(); - tdeCurrentCreateEvent.relation = stmt->into->rel; + if (shouldEncryptTable(stmt->into->accessMethod)) + tdeCurrentCreateEvent.encryptMode = true; - checkEncryptionClause(accessMethod); + checkEncryptionStatus(); } else if (IsA(parsetree, AlterTableStmt)) { - AlterTableStmt *stmt = (AlterTableStmt *) parsetree; + AlterTableStmt *stmt = castNode(AlterTableStmt, parsetree); ListCell *lcmd; - Oid relationId = RangeVarGetRelid(stmt->relation, NoLock, true); + Oid relationId = RangeVarGetRelid(stmt->relation, AccessShareLock, true); + AlterTableCmd *setAccessMethod = NULL; validateCurrentEventTriggerState(true); tdeCurrentCreateEvent.tid = GetCurrentFullTransactionId(); foreach(lcmd, stmt->cmds) { - AlterTableCmd *cmd = (AlterTableCmd *) lfirst(lcmd); + AlterTableCmd *cmd = castNode(AlterTableCmd, lfirst(lcmd)); if (cmd->subtype == AT_SetAccessMethod) - { - const char *accessMethod = cmd->name; - - tdeCurrentCreateEvent.relation = stmt->relation; - tdeCurrentCreateEvent.baseTableOid = relationId; - tdeCurrentCreateEvent.alterAccessMethodMode = true; - - checkEncryptionClause(accessMethod); - alterSetAccessMethod = true; - } + setAccessMethod = cmd; } - if (!alterSetAccessMethod) + tdeCurrentCreateEvent.baseTableOid = relationId; + + /* + * With a SET ACCESS METHOD clause, use that as the basis for + * decisions. But if it's not present, look up encryption status of + * the table. + */ + if (setAccessMethod) { - /* - * With a SET ACCESS METHOD clause, use that as the basis for - * decisions. But if it's not present, look up encryption status - * of the table. - */ + if (shouldEncryptTable(setAccessMethod->name)) + tdeCurrentCreateEvent.encryptMode = true; - tdeCurrentCreateEvent.baseTableOid = relationId; - tdeCurrentCreateEvent.relation = stmt->relation; + checkEncryptionStatus(); + tdeCurrentCreateEvent.alterAccessMethodMode = true; + } + else + { if (relationId != InvalidOid) { - LOCKMODE lockmode = AccessShareLock; - Relation rel = relation_open(relationId, lockmode); + Relation rel = relation_open(relationId, NoLock); if (rel->rd_rel->relam == get_tde_table_am_oid()) { @@ -227,12 +218,11 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) */ tdeCurrentCreateEvent.encryptMode = true; } - relation_close(rel, lockmode); + + relation_close(rel, NoLock); if (tdeCurrentCreateEvent.encryptMode) - { - checkEncryptionClause(""); - } + checkPrincipalKeyConfigured(); } } } @@ -249,7 +239,8 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) reset_current_tde_create_event(); } } - PG_RETURN_NULL(); + + PG_RETURN_VOID(); } /* @@ -262,46 +253,35 @@ pg_tde_ddl_command_end_capture(PG_FUNCTION_ARGS) EventTriggerData *trigdata; Node *parsetree; - trigdata = (EventTriggerData *) fcinfo->context; - parsetree = trigdata->parsetree; - /* Ensure this function is being called as an event trigger */ if (!CALLED_AS_EVENT_TRIGGER(fcinfo)) /* internal error */ ereport(ERROR, - (errmsg("Function can only be fired by event trigger manager"))); + errmsg("Function can only be fired by event trigger manager")); + + trigdata = castNode(EventTriggerData, fcinfo->context); + parsetree = trigdata->parsetree; if (IsA(parsetree, AlterTableStmt) && tdeCurrentCreateEvent.alterAccessMethodMode) { /* - * sequences are not updated automatically call a helper function that - * automatically alters all of them, forcing an update on the - * encryption status int ret; + * sequences are not updated automatically so force rewrite by + * updating their persistence to be the same as before. */ - char *sql = "SELECT pg_tde_internal_refresh_sequences($1);"; - Oid argtypes[1]; - SPIPlanPtr plan; - Datum args[1]; - char nulls[1]; - int ret; + List *seqlist = getOwnedSequences(tdeCurrentCreateEvent.baseTableOid); + ListCell *lc; + Relation rel = relation_open(tdeCurrentCreateEvent.baseTableOid, NoLock); + char persistence = rel->rd_rel->relpersistence; - SPI_connect(); + relation_close(rel, NoLock); - argtypes[0] = OIDOID; - plan = SPI_prepare(sql, 1, argtypes); - - args[0] = ObjectIdGetDatum(tdeCurrentCreateEvent.baseTableOid); - nulls[0] = ' '; + foreach(lc, seqlist) + { + Oid seq_relid = lfirst_oid(lc); - ret = SPI_execute_plan(plan, args, nulls, false, 0); + SequenceChangePersistence(seq_relid, persistence); + } tdeCurrentCreateEvent.alterAccessMethodMode = false; - - SPI_finish(); - - if (ret != SPI_OK_SELECT) - { - elog(ERROR, "Failed to update encryption status of sequences."); - } } /* @@ -315,7 +295,7 @@ pg_tde_ddl_command_end_capture(PG_FUNCTION_ARGS) reset_current_tde_create_event(); } - PG_RETURN_NULL(); + PG_RETURN_VOID(); } static void @@ -323,9 +303,7 @@ reset_current_tde_create_event(void) { tdeCurrentCreateEvent.encryptMode = false; tdeCurrentCreateEvent.baseTableOid = InvalidOid; - tdeCurrentCreateEvent.relation = NULL; tdeCurrentCreateEvent.tid = InvalidFullTransactionId; - alterSetAccessMethod = false; tdeCurrentCreateEvent.alterAccessMethodMode = false; } diff --git a/contrib/pg_tde/src/smgr/pg_tde_smgr.c b/contrib/pg_tde/src/smgr/pg_tde_smgr.c index 2a140c2fd0df2..df097bdd978d5 100644 --- a/contrib/pg_tde/src/smgr/pg_tde_smgr.c +++ b/contrib/pg_tde/src/smgr/pg_tde_smgr.c @@ -26,24 +26,29 @@ typedef struct TDESMgrRelationData typedef TDESMgrRelationData *TDESMgrRelation; +static void CalcBlockIv(ForkNumber forknum, BlockNumber bn, const unsigned char *base_iv, unsigned char *iv); + static InternalKey * -tde_smgr_get_key(SMgrRelation reln, RelFileLocator *old_locator, bool can_create) +tde_smgr_get_key(const RelFileLocatorBackend *smgr_rlocator) +{ + return GetSMGRRelationKey(*smgr_rlocator); +} + +static bool +tde_smgr_should_encrypt(const RelFileLocatorBackend *smgr_rlocator, RelFileLocator *old_locator) { TdeCreateEvent *event; - InternalKey *key; - if (IsCatalogRelationOid(reln->smgr_rlocator.locator.relNumber)) - { - /* do not try to encrypt/decrypt catalog tables */ - return NULL; - } + /* Always encrypt catalog tables */ + if (IsCatalogRelationOid(smgr_rlocator->locator.relNumber)) + return true; - /* see if we have a key for the relation, and return if yes */ - key = GetSMGRRelationKey(reln->smgr_rlocator); - if (key != NULL) - { - return key; - } + /* + * Make sure that even if a statement failed, and an event trigger end + * trigger didn't fire, we don't accidentaly create encrypted files when + * we don't have to. + */ + validateCurrentEventTriggerState(false); event = GetCurrentTdeCreateEvent(); @@ -53,25 +58,22 @@ tde_smgr_get_key(SMgrRelation reln, RelFileLocator *old_locator, bool can_create * * Every file has its own key, that makes logistics easier. */ - if (event->encryptMode == true && can_create) - { - return pg_tde_create_smgr_key(&reln->smgr_rlocator); - } + if (event->encryptMode) + return true; /* check if we had a key for the old locator, if there's one */ - if (old_locator != NULL && can_create) + if (!event->alterAccessMethodMode && old_locator) { - RelFileLocatorBackend rlocator = {.locator = *old_locator,.backend = reln->smgr_rlocator.backend}; - InternalKey *oldkey = GetSMGRRelationKey(rlocator); + RelFileLocatorBackend old_smgr_locator = { + .locator = *old_locator, + .backend = smgr_rlocator->backend, + }; - if (oldkey != NULL) - { - /* create a new key for the new file */ - return pg_tde_create_smgr_key(&reln->smgr_rlocator); - } + if (GetSMGRRelationKey(old_smgr_locator)) + return true; } - return NULL; + return false; } static void @@ -96,12 +98,11 @@ tde_mdwritev(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, for (int i = 0; i < nblocks; ++i) { BlockNumber bn = blocknum + i; - unsigned char iv[16] = {0,}; + unsigned char iv[16]; local_buffers[i] = &local_blocks_aligned[i * BLCKSZ]; - - memcpy(iv + 4, &bn, sizeof(BlockNumber)); + CalcBlockIv(forknum, bn, int_key->base_iv, iv); AesEncrypt(int_key->key, iv, ((unsigned char **) buffers)[i], BLCKSZ, local_buffers[i]); } @@ -114,6 +115,28 @@ tde_mdwritev(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, } } +static void +tde_mdunlink(RelFileLocatorBackend rlocator, ForkNumber forknum, bool isRedo) +{ + mdunlink(rlocator, forknum, isRedo); + + /* + * As of PostgreSQL 17 we are called once per forks, no matter if they + * exist or not, from smgrdounlinkall() so deleting the relation key on + * attempting to delete the main fork is safe. Additionally since we + * unlink the files after commit/abort we do not need to care about + * concurrent accesses. + * + * We support InvalidForkNumber to be similar to mdunlink() but it can + * actually never happen. + */ + if (forknum == MAIN_FORKNUM || forknum == InvalidForkNumber) + { + if (!RelFileLocatorBackendIsTemp(rlocator) && GetSMGRRelationKey(rlocator)) + pg_tde_free_key_map_entry(&rlocator.locator); + } +} + static void tde_mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, const void *buffer, bool skipFsync) @@ -129,12 +152,11 @@ tde_mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, { unsigned char *local_blocks = palloc(BLCKSZ * (1 + 1)); unsigned char *local_blocks_aligned = (unsigned char *) TYPEALIGN(PG_IO_ALIGN_SIZE, local_blocks); - unsigned char iv[16] = { - 0, - }; + unsigned char iv[16]; AesInit(); - memcpy(iv + 4, &blocknum, sizeof(BlockNumber)); + + CalcBlockIv(forknum, blocknum, int_key->base_iv, iv); AesEncrypt(int_key->key, iv, ((unsigned char *) buffer), BLCKSZ, local_blocks_aligned); @@ -162,26 +184,21 @@ tde_mdreadv(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, { bool allZero = true; BlockNumber bn = blocknum + i; - unsigned char iv[16] = {0,}; + unsigned char iv[16]; + /* + * Detect unencrypted all-zero pages written by smgrzeroextend() by + * looking at the first 32 bytes of the page. + * + * Not encrypting all-zero pages is safe because they are only written + * at the end of the file when extending a table on disk so they tend + * to be short lived plus they only leak a slightly more accurate + * table size than one can glean from just the file size. + */ for (int j = 0; j < 32; ++j) { if (((char **) buffers)[i][j] != 0) { - /* - * Postgres creates all zero blocks in an optimized route, - * which we do not try - */ - /* to encrypt. */ - /* - * Instead we detect if a block is all zero at decryption - * time, and - */ - /* leave it as is. */ - /* - * This could be a security issue later, but it is a good - * first prototype - */ allZero = false; break; } @@ -189,7 +206,7 @@ tde_mdreadv(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, if (allZero) continue; - memcpy(iv + 4, &bn, sizeof(BlockNumber)); + CalcBlockIv(forknum, bn, int_key->base_iv, iv); AesDecrypt(int_key->key, iv, ((unsigned char **) buffers)[i], BLCKSZ, ((unsigned char **) buffers)[i]); } @@ -199,16 +216,10 @@ static void tde_mdcreate(RelFileLocator relold, SMgrRelation reln, ForkNumber forknum, bool isRedo) { TDESMgrRelation tdereln = (TDESMgrRelation) reln; - InternalKey *key; - TdeCreateEvent *event = GetCurrentTdeCreateEvent(); - /* - * Make sure that even if a statement failed, and an event trigger end - * trigger didn't fire, we don't accidentaly create encrypted files when - * we don't have to. event above is a pointer, so it will reflect the - * correct state even if this changes it. - */ - validateCurrentEventTriggerState(false); + /* Copied from mdcreate() in md.c */ + if (isRedo && tdereln->md_num_open_segs[forknum] > 0) + return; /* * This is the only function that gets called during actual CREATE @@ -225,13 +236,17 @@ tde_mdcreate(RelFileLocator relold, SMgrRelation reln, ForkNumber forknum, bool * be created later, even during tde creation events. We definitely do * not want to create keys then, even later, when we encrypt all * forks! - */ - - /* + * * Later calls then decide to encrypt or not based on the existence of - * the key + * the key. + * + * Since event triggers do not fire on the standby or in recovery we + * do not try to generate any new keys and instead trust the xlog. */ - key = tde_smgr_get_key(reln, event->alterAccessMethodMode ? NULL : &relold, true); + InternalKey *key = tde_smgr_get_key(&reln->smgr_rlocator); + + if (!isRedo && !key && tde_smgr_should_encrypt(&reln->smgr_rlocator, &relold)) + key = pg_tde_create_smgr_key(&reln->smgr_rlocator); if (key) { @@ -252,7 +267,7 @@ static void tde_mdopen(SMgrRelation reln) { TDESMgrRelation tdereln = (TDESMgrRelation) reln; - InternalKey *key = tde_smgr_get_key(reln, NULL, false); + InternalKey *key = tde_smgr_get_key(&reln->smgr_rlocator); if (key) { @@ -274,7 +289,7 @@ static const struct f_smgr tde_smgr = { .smgr_close = mdclose, .smgr_create = tde_mdcreate, .smgr_exists = mdexists, - .smgr_unlink = mdunlink, + .smgr_unlink = tde_mdunlink, .smgr_extend = tde_mdextend, .smgr_zeroextend = mdzeroextend, .smgr_prefetch = mdprefetch, @@ -294,3 +309,25 @@ RegisterStorageMgr(void) elog(FATAL, "Another storage manager was loaded before pg_tde. Multiple storage managers is unsupported."); storage_manager_id = smgr_register(&tde_smgr, sizeof(TDESMgrRelationData)); } + +/* + * The intialization vector of a block is its block number conmverted to a + * 128 bit big endian number plus the forknumber XOR the base IV of the + * relation file. + */ +static void +CalcBlockIv(ForkNumber forknum, BlockNumber bn, const unsigned char *base_iv, unsigned char *iv) +{ + memset(iv, 0, 16); + + /* The init fork is copied to the main fork so we must use the same IV */ + iv[7] = forknum == INIT_FORKNUM ? MAIN_FORKNUM : forknum; + + iv[12] = bn >> 24; + iv[13] = bn >> 16; + iv[14] = bn >> 8; + iv[15] = bn; + + for (int i = 0; i < 16; i++) + iv[i] ^= base_iv[i]; +} diff --git a/contrib/pg_tde/src/transam/pg_tde_xact_handler.c b/contrib/pg_tde/src/transam/pg_tde_xact_handler.c deleted file mode 100644 index 3e7a8f2385c3f..0000000000000 --- a/contrib/pg_tde/src/transam/pg_tde_xact_handler.c +++ /dev/null @@ -1,187 +0,0 @@ -/*------------------------------------------------------------------------- - * - * pg_tde_xact_handler.c - * Transaction handling routines for pg_tde - * - * - * IDENTIFICATION - * src/transam/pg_tde_xact_handler.c - * - *------------------------------------------------------------------------- - */ - -#include "postgres.h" -#include "access/xact.h" -#include "utils/memutils.h" -#include "utils/palloc.h" -#include "utils/elog.h" -#include "storage/fd.h" -#include "transam/pg_tde_xact_handler.h" -#include "access/pg_tde_tdemap.h" - -typedef struct PendingMapEntryDelete -{ - off_t map_entry_offset; /* map entry offset */ - RelFileLocator rlocator; /* main for use as relation OID */ - bool atCommit; /* T=delete at commit; F=delete at abort */ - int nestLevel; /* xact nesting level of request */ - struct PendingMapEntryDelete *next; /* linked-list link */ -} PendingMapEntryDelete; - -static PendingMapEntryDelete *pendingDeletes = NULL; /* head of linked list */ - -static void do_pending_deletes(bool isCommit); -static void reassign_pending_deletes_to_parent_xact(void); -static void pending_delete_cleanup(void); - -/* Transaction Callbacks from Backend*/ -static void -pg_tde_xact_callback(XactEvent event, void *arg) -{ - if (event == XACT_EVENT_PARALLEL_ABORT || - event == XACT_EVENT_ABORT) - { - ereport(DEBUG2, - (errmsg("pg_tde_xact_callback: aborting transaction"))); - do_pending_deletes(false); - } - else if (event == XACT_EVENT_COMMIT) - { - do_pending_deletes(true); - pending_delete_cleanup(); - } - else if (event == XACT_EVENT_PREPARE) - { - pending_delete_cleanup(); - } -} - -static void -pg_tde_subxact_callback(SubXactEvent event, SubTransactionId mySubid, - SubTransactionId parentSubid, void *arg) -{ - /* TODO: takle all possible transaction states */ - if (event == SUBXACT_EVENT_ABORT_SUB) - { - ereport(DEBUG2, - (errmsg("pg_tde_subxact_callback: aborting subtransaction"))); - do_pending_deletes(false); - } - else if (event == SUBXACT_EVENT_COMMIT_SUB) - { - ereport(DEBUG2, - (errmsg("pg_tde_subxact_callback: committing subtransaction"))); - reassign_pending_deletes_to_parent_xact(); - } -} - -void -RegisterTdeXactCallbacks(void) -{ - RegisterXactCallback(pg_tde_xact_callback, NULL); - RegisterSubXactCallback(pg_tde_subxact_callback, NULL); -} - -void -RegisterEntryForDeletion(const RelFileLocator *rlocator, off_t map_entry_offset, bool atCommit) -{ - PendingMapEntryDelete *pending; - - pending = (PendingMapEntryDelete *) MemoryContextAlloc(TopMemoryContext, sizeof(PendingMapEntryDelete)); - pending->map_entry_offset = map_entry_offset; - pending->rlocator = *rlocator; - pending->atCommit = atCommit; /* delete if abort */ - pending->nestLevel = GetCurrentTransactionNestLevel(); - pending->next = pendingDeletes; - pendingDeletes = pending; -} - -/* - * do_pending_deletes() -- Take care of file deletes at end of xact. - * - * This also runs when aborting a subxact; we want to clean up a failed - * subxact immediately. - * - */ -static void -do_pending_deletes(bool isCommit) -{ - int nestLevel = GetCurrentTransactionNestLevel(); - PendingMapEntryDelete *pending; - PendingMapEntryDelete *prev; - PendingMapEntryDelete *next; - - prev = NULL; - for (pending = pendingDeletes; pending != NULL; pending = next) - { - next = pending->next; - if (pending->nestLevel != nestLevel) - { - /* outer-level entries should not be processed yet */ - prev = pending; - continue; - } - - /* unlink list entry first, so we don't retry on failure */ - if (prev) - prev->next = next; - else - pendingDeletes = next; - /* do deletion if called for */ - if (pending->atCommit == isCommit) - { - ereport(LOG, - (errmsg("pg_tde_xact_callback: deleting entry at offset %d", - (int) (pending->map_entry_offset)))); - pg_tde_free_key_map_entry(&pending->rlocator, pending->map_entry_offset); - } - pfree(pending); - /* prev does not change */ - - } -} - - -/* - * reassign_pending_deletes_to_parent_xact() -- Adjust nesting level of pending deletes. - * - * There are several cases to consider: - * 1. Only top level transaction can perform on-commit deletes. - * 2. Subtransaction and top level transaction can perform on-abort deletes. - * So we have to decrement the nesting level of pending deletes to reassing them to the parent transaction - * if subtransaction was not self aborted. In other words if subtransaction state is commited all its pending - * deletes are reassigned to the parent transaction. - */ -static void -reassign_pending_deletes_to_parent_xact(void) -{ - PendingMapEntryDelete *pending; - int nestLevel = GetCurrentTransactionNestLevel(); - - for (pending = pendingDeletes; pending != NULL; pending = pending->next) - { - if (pending->nestLevel == nestLevel) - pending->nestLevel--; - } -} - -/* - * pending_delete_cleanup -- Clean up after a successful PREPARE or COMMIT - * - * What we have to do here is throw away the in-memory state about pending - * file deletes. It's all been recorded in the 2PC state file and - * it's no longer our job to worry about it. - */ -static void -pending_delete_cleanup(void) -{ - PendingMapEntryDelete *pending; - PendingMapEntryDelete *next; - - for (pending = pendingDeletes; pending != NULL; pending = next) - { - next = pending->next; - pendingDeletes = next; - pfree(pending); - } -} diff --git a/contrib/pg_tde/t/001_basic.pl b/contrib/pg_tde/t/001_basic.pl index 8a6cf445f2cae..c9619b104e9f2 100644 --- a/contrib/pg_tde/t/001_basic.pl +++ b/contrib/pg_tde/t/001_basic.pl @@ -3,96 +3,77 @@ use strict; use warnings; use File::Basename; -use File::Compare; -use File::Copy; use Test::More; use lib 't'; use pgtde; -# Get file name and CREATE out file name and dirs WHERE requried PGTDE::setup_files_dir(basename($0)); -# CREATE new PostgreSQL node and do initdb -my $node = PGTDE->pgtde_init_pg(); -my $pgdata = $node->data_dir; +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf('postgresql.conf', "shared_preload_libraries = 'pg_tde'"); +$node->start; -# UPDATE postgresql.conf to include/load pg_tde library -open my $conf, '>>', "$pgdata/postgresql.conf"; -print $conf "shared_preload_libraries = 'pg_tde'\n"; -close $conf; +PGTDE::psql($node, 'postgres', 'CREATE EXTENSION IF NOT EXISTS pg_tde;'); -# Start server -my $rt_value = $node->start; -ok($rt_value == 1, "Start Server"); +PGTDE::psql($node, 'postgres', + 'SELECT extname, extversion FROM pg_extension WHERE extname = \'pg_tde\';' +); -# CREATE EXTENSION IF NOT EXISTS and change out file permissions -my ($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS pg_tde;', extra_params => ['-a']); -ok($cmdret == 0, "CREATE PGTDE EXTENSION"); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', + 'CREATE TABLE test_enc(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap;' +); -my ($cmdret, $stdout, $stderr) = $node->psql('postgres', 'SELECT extname, extversion FROM pg_extension WHERE extname = \'pg_tde\';', extra_params => ['-a']); -ok($cmdret == 0, "SELECT PGTDE VERSION"); -PGTDE::append_to_file($stdout); +PGTDE::append_to_result_file("-- server restart"); +$node->restart; -$rt_value = $node->psql('postgres', 'CREATE TABLE test_enc(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap;', extra_params => ['-a']); -ok($rt_value == 3, "Failing query"); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');" +); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault');" +); -# Restart the server -PGTDE::append_to_file("-- server restart"); -$node->stop(); +PGTDE::psql($node, 'postgres', + 'CREATE TABLE test_enc(id SERIAL,k VARCHAR(32),PRIMARY KEY (id)) USING tde_heap;' +); -$rt_value = $node->start(); -ok($rt_value == 1, "Restart Server"); +PGTDE::psql($node, 'postgres', + 'INSERT INTO test_enc (k) VALUES (\'foobar\'),(\'barfoo\');'); -$rt_value = $node->psql('postgres', "SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');", extra_params => ['-a']); -$rt_value = $node->psql('postgres', "SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault');", extra_params => ['-a']); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc ORDER BY id ASC;'); -$stdout = $node->safe_psql('postgres', 'CREATE TABLE test_enc(id SERIAL,k VARCHAR(32),PRIMARY KEY (id)) USING tde_heap;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::append_to_result_file("-- server restart"); +$node->restart; -$stdout = $node->safe_psql('postgres', 'INSERT INTO test_enc (k) VALUES (\'foobar\'),(\'barfoo\');', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -# Restart the server -PGTDE::append_to_file("-- server restart"); -$rt_value = $node->stop(); -$rt_value = $node->start(); - -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc ORDER BY id ASC;'); # Verify that we can't see the data in the file my $tablefile = $node->safe_psql('postgres', 'SHOW data_directory;'); $tablefile .= '/'; -$tablefile .= $node->safe_psql('postgres', 'SELECT pg_relation_filepath(\'test_enc\');'); +$tablefile .= + $node->safe_psql('postgres', 'SELECT pg_relation_filepath(\'test_enc\');'); my $strings = 'TABLEFILE FOUND: '; $strings .= `(ls $tablefile >/dev/null && echo yes) || echo no`; -PGTDE::append_to_file($strings); +PGTDE::append_to_result_file($strings); $strings = 'CONTAINS FOO (should be empty): '; $strings .= `strings $tablefile | grep foo`; -PGTDE::append_to_file($strings); +PGTDE::append_to_result_file($strings); + +PGTDE::psql($node, 'postgres', 'DROP TABLE test_enc;'); -$stdout = $node->safe_psql('postgres', 'DROP TABLE test_enc;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'DROP EXTENSION pg_tde;'); -# DROP EXTENSION -$stdout = $node->safe_psql('postgres', 'DROP EXTENSION pg_tde;', extra_params => ['-a']); -ok($cmdret == 0, "DROP PGTDE EXTENSION"); -PGTDE::append_to_file($stdout); -# Stop the server -$node->stop(); +$node->stop; -# compare the expected and out file +# Compare the expected and out file my $compare = PGTDE->compare_results(); -# Test/check if expected and result/out file match. If Yes, test passes. -is($compare,0,"Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files."); +is($compare, 0, + "Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files." +); -# Done testing for this testcase file. done_testing(); diff --git a/contrib/pg_tde/t/002_rotate_key.pl b/contrib/pg_tde/t/002_rotate_key.pl index ed744a578d6a1..f590a81ac4be6 100644 --- a/contrib/pg_tde/t/002_rotate_key.pl +++ b/contrib/pg_tde/t/002_rotate_key.pl @@ -3,199 +3,169 @@ use strict; use warnings; use File::Basename; -use File::Compare; -use File::Copy; use Test::More; use lib 't'; use pgtde; -# Get file name and CREATE out file name and dirs WHERE requried PGTDE::setup_files_dir(basename($0)); -# CREATE new PostgreSQL node and do initdb -my $node = PGTDE->pgtde_init_pg(); -my $pgdata = $node->data_dir; - -# UPDATE postgresql.conf to include/load pg_tde library -open my $conf, '>>', "$pgdata/postgresql.conf"; -print $conf "shared_preload_libraries = 'pg_tde'\n"; -close $conf; - -# Start server -my $rt_value = $node->start; -ok($rt_value == 1, "Start Server"); - -# CREATE EXTENSION IF NOT EXISTS and change out file permissions -my ($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS pg_tde;', extra_params => ['-a']); -ok($cmdret == 0, "CREATE PGTDE EXTENSION"); -PGTDE::append_to_file($stdout); - - -$rt_value = $node->psql('postgres', 'CREATE TABLE test_enc(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap;', extra_params => ['-a']); -ok($rt_value == 3, "Failing query"); - - -# Restart the server -PGTDE::append_to_file("-- server restart"); -$node->stop(); - -$rt_value = $node->start(); -ok($rt_value == 1, "Restart Server"); - -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_add_key_provider_file('file-2','/tmp/pg_tde_test_keyring_2.per');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_add_global_key_provider_file('file-2','/tmp/pg_tde_test_keyring_2g.per');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_add_global_key_provider_file('file-3','/tmp/pg_tde_test_keyring_3.per');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_list_all_key_providers();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -$stdout = $node->safe_psql('postgres', 'CREATE TABLE test_enc(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -$stdout = $node->safe_psql('postgres', 'INSERT INTO test_enc (k) VALUES (5),(6);', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -#rotate key -$stdout = $node->psql('postgres', "SELECT pg_tde_set_principal_key('rotated-principal-key1');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -# Restart the server -PGTDE::append_to_file("-- server restart"); -$rt_value = $node->stop(); -$rt_value = $node->start(); - -$stdout = $node->safe_psql('postgres', "SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_principal_key_info();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -($cmdret, $stdout, $stderr) = $node->psql('postgres', "SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_global_principal_key_info();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -PGTDE::append_to_file($stderr); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - - -#Again rotate key -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_set_principal_key('rotated-principal-key2','file-2');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -# Restart the server -PGTDE::append_to_file("-- server restart"); -$rt_value = $node->stop(); -$rt_value = $node->start(); - -$stdout = $node->safe_psql('postgres', "SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_principal_key_info();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -($cmdret, $stdout, $stderr) = $node->psql('postgres', "SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_global_principal_key_info();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -PGTDE::append_to_file($stderr); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -#Again rotate key -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_set_global_principal_key('rotated-principal-key', 'file-3', false);", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -# Restart the server -PGTDE::append_to_file("-- server restart"); -$rt_value = $node->stop(); -$rt_value = $node->start(); - -$stdout = $node->safe_psql('postgres', "SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_principal_key_info();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -($cmdret, $stdout, $stderr) = $node->psql('postgres', "SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_global_principal_key_info();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -PGTDE::append_to_file($stderr); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf('postgresql.conf', "shared_preload_libraries = 'pg_tde'"); +$node->start; + +PGTDE::psql($node, 'postgres', 'CREATE EXTENSION IF NOT EXISTS pg_tde;'); + +PGTDE::psql($node, 'postgres', + 'CREATE TABLE test_enc(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap;' +); + +PGTDE::append_to_result_file("-- server restart"); +$node->restart; + +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');" +); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_add_database_key_provider_file('file-2','/tmp/pg_tde_test_keyring_2.per');" +); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_add_global_key_provider_file('file-2','/tmp/pg_tde_test_keyring_2g.per');" +); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_add_global_key_provider_file('file-3','/tmp/pg_tde_test_keyring_3.per');" +); + +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_list_all_database_key_providers();"); + +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault');" +); + +PGTDE::psql($node, 'postgres', + 'CREATE TABLE test_enc(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap;' +); + +PGTDE::psql($node, 'postgres', 'INSERT INTO test_enc (k) VALUES (5),(6);'); + +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc ORDER BY id ASC;'); + +# Rotate key +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_key_using_database_key_provider('rotated-key1');"); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc ORDER BY id ASC;'); + +PGTDE::append_to_result_file("-- server restart"); +$node->restart; + +PGTDE::psql($node, 'postgres', + "SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_key_info();" +); +PGTDE::psql($node, 'postgres', + "SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_server_key_info();" +); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc ORDER BY id ASC;'); + +# Again rotate key +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_key_using_database_key_provider('rotated-key2','file-2');" +); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc ORDER BY id ASC;'); + +PGTDE::append_to_result_file("-- server restart"); +$node->restart; + +PGTDE::psql($node, 'postgres', + "SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_key_info();" +); +PGTDE::psql($node, 'postgres', + "SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_server_key_info();" +); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc ORDER BY id ASC;'); + +# Again rotate key +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_key_using_global_key_provider('rotated-key', 'file-3', false);" +); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc ORDER BY id ASC;'); + +PGTDE::append_to_result_file("-- server restart"); +$node->restart; + +PGTDE::psql($node, 'postgres', + "SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_key_info();" +); +PGTDE::psql($node, 'postgres', + "SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_server_key_info();" +); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc ORDER BY id ASC;'); # TODO: add method to query current info # And maybe debug tools to show what's in a file keyring? -#Again rotate key -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_set_global_principal_key('rotated-principal-keyX', 'file-2', false);", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +# Again rotate key +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_key_using_global_key_provider('rotated-keyX', 'file-2', false);" +); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc ORDER BY id ASC;'); -# Restart the server -PGTDE::append_to_file("-- server restart"); -$rt_value = $node->stop(); -$rt_value = $node->start(); +PGTDE::append_to_result_file("-- server restart"); +$node->restart; -$stdout = $node->safe_psql('postgres', "SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_principal_key_info();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -($cmdret, $stdout, $stderr) = $node->psql('postgres', "SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_global_principal_key_info();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -PGTDE::append_to_file($stderr); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', + "SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_key_info();" +); +PGTDE::psql($node, 'postgres', + "SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_server_key_info();" +); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc ORDER BY id ASC;'); -$stdout = $node->safe_psql('postgres', 'ALTER SYSTEM SET pg_tde.inherit_global_providers = OFF;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', + 'ALTER SYSTEM SET pg_tde.inherit_global_providers = OFF;'); # Things still work after a restart -# Restart the server -PGTDE::append_to_file("-- server restart"); -$rt_value = $node->stop(); -$rt_value = $node->start(); +PGTDE::append_to_result_file("-- server restart"); +$node->restart; # But now can't be changed to another global provider -($cmdret, $stdout, $stderr) = $node->psql('postgres', "SELECT pg_tde_set_global_principal_key('rotated-principal-keyX2', 'file-2', false);", extra_params => ['-a']); -PGTDE::append_to_file($stderr); -$stdout = $node->safe_psql('postgres', "SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_principal_key_info();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -($cmdret, $stdout, $stderr) = $node->psql('postgres', "SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_global_principal_key_info();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -PGTDE::append_to_file($stderr); - -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_set_principal_key('rotated-principal-key2','file-2');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', "SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_principal_key_info();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -($cmdret, $stdout, $stderr) = $node->psql('postgres', "SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_global_principal_key_info();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -PGTDE::append_to_file($stderr); - - -# end - -$stdout = $node->safe_psql('postgres', 'DROP TABLE test_enc;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -$stdout = $node->safe_psql('postgres', 'ALTER SYSTEM RESET pg_tde.inherit_global_providers;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -$rt_value = $node->stop(); -$rt_value = $node->start(); - -# DROP EXTENSION -($cmdret, $stdout, $stderr) = $node->psql('postgres', 'DROP EXTENSION pg_tde CASCADE;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); -PGTDE::append_to_file($stderr); -# Stop the server -$node->stop(); - -# compare the expected and out file +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_key_using_global_key_provider('rotated-keyX2', 'file-2', false);" +); +PGTDE::psql($node, 'postgres', + "SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_key_info();" +); +PGTDE::psql($node, 'postgres', + "SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_server_key_info();" +); + +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_key_using_database_key_provider('rotated-key2','file-2');" +); +PGTDE::psql($node, 'postgres', + "SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_key_info();" +); +PGTDE::psql($node, 'postgres', + "SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_server_key_info();" +); + +PGTDE::psql($node, 'postgres', 'DROP TABLE test_enc;'); + +PGTDE::psql($node, 'postgres', + 'ALTER SYSTEM RESET pg_tde.inherit_global_providers;'); + +PGTDE::append_to_result_file("-- server restart"); +$node->restart; + +PGTDE::psql($node, 'postgres', 'DROP EXTENSION pg_tde CASCADE;'); + +$node->stop; + +# Compare the expected and out file my $compare = PGTDE->compare_results(); -# Test/check if expected and result/out file match. If Yes, test passes. -is($compare,0,"Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files."); +is($compare, 0, + "Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files." +); -# Done testing for this testcase file. done_testing(); diff --git a/contrib/pg_tde/t/003_remote_config.pl b/contrib/pg_tde/t/003_remote_config.pl index 3c2394236e07f..1dc06e6f8ec13 100644 --- a/contrib/pg_tde/t/003_remote_config.pl +++ b/contrib/pg_tde/t/003_remote_config.pl @@ -3,110 +3,95 @@ use strict; use warnings; use File::Basename; -use File::Compare; -use File::Copy; use Test::More; use lib 't'; use pgtde; -# Get file name and CREATE out file name and dirs WHERE requried -PGTDE::setup_files_dir(basename($0)); +{ -# CREATE new PostgreSQL node and do initdb -my $node = PGTDE->pgtde_init_pg(); -my $pgdata = $node->data_dir; + package MyWebServer; -{ -package MyWebServer; - -use HTTP::Server::Simple::CGI; -use base qw(HTTP::Server::Simple::CGI); - -my %dispatch = ( - '/hello' => \&resp_hello, - # ... -); - -sub handle_request { - my $self = shift; - my $cgi = shift; - - my $path = $cgi->path_info(); - my $handler = $dispatch{$path}; - - if (ref($handler) eq "CODE") { - print "HTTP/1.0 200 OK\r\n"; - $handler->($cgi); - - } else { - print "HTTP/1.0 404 Not found\r\n"; - print $cgi->header, - $cgi->start_html('Not found'), - $cgi->h1('Not found'), - $cgi->end_html; - } -} + use HTTP::Server::Simple::CGI; + use base qw(HTTP::Server::Simple::CGI); + + my %dispatch = ('/hello' => \&resp_hello,); + + sub handle_request + { + my $self = shift; + my $cgi = shift; + + my $path = $cgi->path_info(); + my $handler = $dispatch{$path}; + + if (ref($handler) eq "CODE") + { + print "HTTP/1.0 200 OK\r\n"; + $handler->($cgi); + + } + else + { + print "HTTP/1.0 404 Not found\r\n"; + print $cgi->header, + $cgi->start_html('Not found'), + $cgi->h1('Not found'), + $cgi->end_html; + } + } + + sub resp_hello + { + my $cgi = shift; + print $cgi->header, "/tmp/http_datafile\r\n"; + } -sub resp_hello { - my $cgi = shift; - print $cgi->header, - "/tmp/http_datafile\r\n"; -} - } + my $pid = MyWebServer->new(8888)->background(); +PGTDE::setup_files_dir(basename($0)); -# UPDATE postgresql.conf to include/load pg_tde library -open my $conf, '>>', "$pgdata/postgresql.conf"; -print $conf "shared_preload_libraries = 'pg_tde'\n"; -close $conf; +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf('postgresql.conf', "shared_preload_libraries = 'pg_tde'"); +$node->start; -my $rt_value = $node->stop(); -$rt_value = $node->start(); -ok($rt_value == 1, "Restart Server"); +PGTDE::psql($node, 'postgres', 'CREATE EXTENSION IF NOT EXISTS pg_tde;'); -my ($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS pg_tde;', extra_params => ['-a']); -ok($cmdret == 0, "CREATE PGTDE EXTENSION"); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_add_database_key_provider_file('file-provider', json_object( 'type' VALUE 'remote', 'url' VALUE 'http://localhost:8888/hello' ));" +); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-provider');" +); -$rt_value = $node->psql('postgres', "SELECT pg_tde_add_key_provider_file('file-provider', json_object( 'type' VALUE 'remote', 'url' VALUE 'http://localhost:8888/hello' ));", extra_params => ['-a']); -$rt_value = $node->psql('postgres', "SELECT pg_tde_set_principal_key('test-db-principal-key','file-provider');", extra_params => ['-a']); +PGTDE::psql($node, 'postgres', + 'CREATE TABLE test_enc2(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap;' +); -$stdout = $node->safe_psql('postgres', 'CREATE TABLE test_enc2(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'INSERT INTO test_enc2 (k) VALUES (5),(6);'); -$stdout = $node->safe_psql('postgres', 'INSERT INTO test_enc2 (k) VALUES (5),(6);', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc2 ORDER BY id ASC;'); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc2 ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::append_to_result_file("-- server restart"); +$node->restart; -# Restart the server -PGTDE::append_to_file("-- server restart"); -$rt_value = $node->stop(); -$rt_value = $node->start(); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc2 ORDER BY id ASC;'); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc2 ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'DROP TABLE test_enc2;'); -$stdout = $node->safe_psql('postgres', 'DROP TABLE test_enc2;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'DROP EXTENSION pg_tde;'); -# DROP EXTENSION -$stdout = $node->safe_psql('postgres', 'DROP EXTENSION pg_tde;', extra_params => ['-a']); -ok($cmdret == 0, "DROP PGTDE EXTENSION"); -PGTDE::append_to_file($stdout); -# Stop the server -$node->stop(); +$node->stop; system("kill $pid"); -# compare the expected and out file +# Compare the expected and out file my $compare = PGTDE->compare_results(); -# Test/check if expected and result/out file match. If Yes, test passes. -is($compare,0,"Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files."); +is($compare, 0, + "Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files." +); -# Done testing for this testcase file. done_testing(); diff --git a/contrib/pg_tde/t/004_file_config.pl b/contrib/pg_tde/t/004_file_config.pl index 478d6bd177f4f..948b4d345976d 100644 --- a/contrib/pg_tde/t/004_file_config.pl +++ b/contrib/pg_tde/t/004_file_config.pl @@ -3,72 +3,54 @@ use strict; use warnings; use File::Basename; -use File::Compare; -use File::Copy; use Test::More; use lib 't'; use pgtde; -# Get file name and CREATE out file name and dirs WHERE requried PGTDE::setup_files_dir(basename($0)); -# CREATE new PostgreSQL node and do initdb -my $node = PGTDE->pgtde_init_pg(); -my $pgdata = $node->data_dir; - -copy("$pgdata/postgresql.conf", "$pgdata/postgresql.conf.bak"); - -# UPDATE postgresql.conf to include/load pg_tde library -open my $conf, '>>', "$pgdata/postgresql.conf"; -print $conf "shared_preload_libraries = 'pg_tde'\n"; -close $conf; - open my $conf2, '>>', "/tmp/datafile-location"; print $conf2 "/tmp/keyring_data_file\n"; close $conf2; -my $rt_value = $node->start(); -ok($rt_value == 1, "Start Server"); +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf('postgresql.conf', "shared_preload_libraries = 'pg_tde'"); +$node->start; + +PGTDE::psql($node, 'postgres', 'CREATE EXTENSION IF NOT EXISTS pg_tde;'); -my ($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS pg_tde;', extra_params => ['-a']); -ok($cmdret == 0, "CREATE PGTDE EXTENSION"); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_add_database_key_provider_file('file-provider', json_object( 'type' VALUE 'file', 'path' VALUE '/tmp/datafile-location' ));" +); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-provider');" +); -$rt_value = $node->psql('postgres', "SELECT pg_tde_add_key_provider_file('file-provider', json_object( 'type' VALUE 'file', 'path' VALUE '/tmp/datafile-location' ));", extra_params => ['-a']); -$rt_value = $node->psql('postgres', "SELECT pg_tde_set_principal_key('test-db-principal-key','file-provider');", extra_params => ['-a']); +PGTDE::psql($node, 'postgres', + 'CREATE TABLE test_enc1(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap;' +); -$stdout = $node->safe_psql('postgres', 'CREATE TABLE test_enc1(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'INSERT INTO test_enc1 (k) VALUES (5),(6);'); -$stdout = $node->safe_psql('postgres', 'INSERT INTO test_enc1 (k) VALUES (5),(6);', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc1 ORDER BY id ASC;'); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc1 ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::append_to_result_file("-- server restart"); +$node->restart; -# Restart the server -PGTDE::append_to_file("-- server restart"); -$rt_value = $node->stop(); -$rt_value = $node->start(); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc1 ORDER BY id ASC;'); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc1 ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'DROP TABLE test_enc1;'); -$stdout = $node->safe_psql('postgres', 'DROP TABLE test_enc1;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'DROP EXTENSION pg_tde;'); -# DROP EXTENSION -$stdout = $node->safe_psql('postgres', 'DROP EXTENSION pg_tde;', extra_params => ['-a']); -ok($cmdret == 0, "DROP PGTDE EXTENSION"); -PGTDE::append_to_file($stdout); -# Stop the server -$node->stop(); +$node->stop; -# compare the expected and out file +# Compare the expected and out file my $compare = PGTDE->compare_results(); -# Test/check if expected and result/out file match. If Yes, test passes. -is($compare,0,"Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files."); +is($compare, 0, + "Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files." +); -# Done testing for this testcase file. done_testing(); diff --git a/contrib/pg_tde/t/005_multiple_extensions.pl b/contrib/pg_tde/t/005_multiple_extensions.pl index e9aff281e4466..3230ab4a0171d 100644 --- a/contrib/pg_tde/t/005_multiple_extensions.pl +++ b/contrib/pg_tde/t/005_multiple_extensions.pl @@ -3,119 +3,173 @@ use strict; use warnings; use File::Basename; -use File::Compare; -use File::Copy; use Test::More; use lib 't'; use pgtde; -# Get file name and CREATE out file name and dirs WHERE requried PGTDE::setup_files_dir(basename($0)); my $PG_VERSION_STRING = `pg_config --version`; if (index(lc($PG_VERSION_STRING), lc("Percona Distribution")) == -1) { - plan skip_all => "pg_tde test case only for PPG server package install with extensions."; + plan skip_all => + "pg_tde test case only for PPG server package install with extensions."; } -# CREATE new PostgreSQL node and do initdb -my $node = PGTDE->pgtde_init_pg(); -my $pgdata = $node->data_dir; - -copy("$pgdata/postgresql.conf", "$pgdata/postgresql.conf.bak"); - -# UPDATE postgresql.conf to include/load pg_stat_monitor library -open my $conf, '>>', "$pgdata/postgresql.conf"; -print $conf "shared_preload_libraries = 'pg_tde, pg_stat_monitor, pgaudit, set_user, pg_repack'\n"; -print $conf "pg_stat_monitor.pgsm_bucket_time = 360000\n"; -print $conf "pg_stat_monitor.pgsm_normalized_query = 'yes'\n"; -close $conf; - open my $conf2, '>>', "/tmp/datafile-location"; print $conf2 "/tmp/keyring_data_file\n"; close $conf2; -# Start server -my $rt_value = $node->start; -ok($rt_value == 1, "Start Server"); +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf('postgresql.conf', + "shared_preload_libraries = 'pg_tde, pg_stat_monitor, pgaudit, set_user, pg_repack'" +); +$node->append_conf('postgresql.conf', + "pg_stat_monitor.pgsm_bucket_time = 360000"); +$node->append_conf('postgresql.conf', + "pg_stat_monitor.pgsm_normalized_query = 'yes'"); +$node->start; # Create PGSM extension -my ($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS pg_stat_monitor;', extra_params => ['-a']); +my ($cmdret, $stdout, $stderr) = $node->psql( + 'postgres', + 'CREATE EXTENSION IF NOT EXISTS pg_stat_monitor;', + extra_params => ['-a']); ok($cmdret == 0, "CREATE PGSM EXTENSION"); PGTDE::append_to_debug_file($stdout); -($cmdret, $stdout, $stderr) = $node->psql('postgres', 'SELECT pg_stat_monitor_reset();', extra_params => ['-a', '-Pformat=aligned','-Ptuples_only=off']); +($cmdret, $stdout, $stderr) = $node->psql( + 'postgres', + 'SELECT pg_stat_monitor_reset();', + extra_params => [ '-a', '-Pformat=aligned', '-Ptuples_only=off' ]); ok($cmdret == 0, "Reset PGSM EXTENSION"); PGTDE::append_to_debug_file($stdout); # Create pg_tde extension -($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS pg_tde;', extra_params => ['-a']); +($cmdret, $stdout, $stderr) = $node->psql( + 'postgres', + 'CREATE EXTENSION IF NOT EXISTS pg_tde;', + extra_params => ['-a']); ok($cmdret == 0, "CREATE PGTDE EXTENSION"); -PGTDE::append_to_file($stdout); +PGTDE::append_to_result_file($stdout); # Create Other extensions -($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS IF NOT EXISTS pgaudit;', extra_params => ['-a']); +($cmdret, $stdout, $stderr) = $node->psql( + 'postgres', + 'CREATE EXTENSION IF NOT EXISTS IF NOT EXISTS pgaudit;', + extra_params => ['-a']); ok($cmdret == 0, "CREATE pgaudit EXTENSION"); PGTDE::append_to_debug_file($stdout); -($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS IF NOT EXISTS set_user;', extra_params => ['-a']); +($cmdret, $stdout, $stderr) = $node->psql( + 'postgres', + 'CREATE EXTENSION IF NOT EXISTS IF NOT EXISTS set_user;', + extra_params => ['-a']); ok($cmdret == 0, "CREATE set_user EXTENSION"); PGTDE::append_to_debug_file($stdout); -($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS IF NOT EXISTS pg_repack;', extra_params => ['-a']); +($cmdret, $stdout, $stderr) = $node->psql( + 'postgres', + 'CREATE EXTENSION IF NOT EXISTS IF NOT EXISTS pg_repack;', + extra_params => ['-a']); ok($cmdret == 0, "CREATE pg_repack EXTENSION"); PGTDE::append_to_debug_file($stdout); -($cmdret, $stdout, $stderr) = $node->psql('postgres', "SET pgaudit.log = 'none'; CREATE EXTENSION IF NOT EXISTS IF NOT EXISTS postgis; SET pgaudit.log = 'all';", extra_params => ['-a']); +($cmdret, $stdout, $stderr) = $node->psql( + 'postgres', + "SET pgaudit.log = 'none'; CREATE EXTENSION IF NOT EXISTS IF NOT EXISTS postgis; SET pgaudit.log = 'all';", + extra_params => ['-a']); ok($cmdret == 0, "CREATE postgis EXTENSION"); PGTDE::append_to_debug_file($stdout); -($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS IF NOT EXISTS postgis_raster;', extra_params => ['-a']); +($cmdret, $stdout, $stderr) = $node->psql( + 'postgres', + 'CREATE EXTENSION IF NOT EXISTS IF NOT EXISTS postgis_raster;', + extra_params => ['-a']); ok($cmdret == 0, "CREATE postgis_raster EXTENSION"); PGTDE::append_to_debug_file($stdout); -($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS IF NOT EXISTS postgis_sfcgal;', extra_params => ['-a']); +($cmdret, $stdout, $stderr) = $node->psql( + 'postgres', + 'CREATE EXTENSION IF NOT EXISTS IF NOT EXISTS postgis_sfcgal;', + extra_params => ['-a']); ok($cmdret == 0, "CREATE postgis_sfcgal EXTENSION"); PGTDE::append_to_debug_file($stdout); -($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS IF NOT EXISTS fuzzystrmatch;', extra_params => ['-a']); +($cmdret, $stdout, $stderr) = $node->psql( + 'postgres', + 'CREATE EXTENSION IF NOT EXISTS IF NOT EXISTS fuzzystrmatch;', + extra_params => ['-a']); ok($cmdret == 0, "CREATE fuzzystrmatch EXTENSION"); PGTDE::append_to_debug_file($stdout); -($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS IF NOT EXISTS address_standardizer;', extra_params => ['-a']); +($cmdret, $stdout, $stderr) = $node->psql( + 'postgres', + 'CREATE EXTENSION IF NOT EXISTS IF NOT EXISTS address_standardizer;', + extra_params => ['-a']); ok($cmdret == 0, "CREATE address_standardizer EXTENSION"); PGTDE::append_to_debug_file($stdout); -($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS IF NOT EXISTS address_standardizer_data_us;', extra_params => ['-a']); +($cmdret, $stdout, $stderr) = $node->psql( + 'postgres', + 'CREATE EXTENSION IF NOT EXISTS IF NOT EXISTS address_standardizer_data_us;', + extra_params => ['-a']); ok($cmdret == 0, "CREATE address_standardizer_data_us EXTENSION"); PGTDE::append_to_debug_file($stdout); -($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS IF NOT EXISTS postgis_tiger_geocoder;', extra_params => ['-a']); +($cmdret, $stdout, $stderr) = $node->psql( + 'postgres', + 'CREATE EXTENSION IF NOT EXISTS IF NOT EXISTS postgis_tiger_geocoder;', + extra_params => ['-a']); ok($cmdret == 0, "CREATE postgis_tiger_geocoder EXTENSION"); PGTDE::append_to_debug_file($stdout); -$rt_value = $node->psql('postgres', "SELECT pg_tde_add_key_provider_file('file-provider', json_object( 'type' VALUE 'file', 'path' VALUE '/tmp/datafile-location' ));", extra_params => ['-a']); -$rt_value = $node->psql('postgres', "SELECT pg_tde_set_principal_key('test-db-principal-key','file-provider');", extra_params => ['-a']); - -$stdout = $node->safe_psql('postgres', 'CREATE TABLE test_enc1(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -$stdout = $node->safe_psql('postgres', 'INSERT INTO test_enc1 (k) VALUES (5),(6);', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc1 ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -# Restart the server -PGTDE::append_to_file("-- server restart"); -$rt_value = $node->stop(); -$rt_value = $node->start(); - -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc1 ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -$stdout = $node->safe_psql('postgres', 'DROP TABLE test_enc1;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -# Print PGSM settings -($cmdret, $stdout, $stderr) = $node->psql('postgres', "SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_query_shared_buffer';", extra_params => ['-a', '-Pformat=aligned','-Ptuples_only=off']); +$node->psql( + 'postgres', + "SELECT pg_tde_add_database_key_provider_file('file-provider', json_object( 'type' VALUE 'file', 'path' VALUE '/tmp/datafile-location' ));", + extra_params => ['-a']); +$node->psql( + 'postgres', + "SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-provider');", + extra_params => ['-a']); + +$stdout = $node->safe_psql( + 'postgres', + 'CREATE TABLE test_enc1(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap;', + extra_params => ['-a']); +PGTDE::append_to_result_file($stdout); + +$stdout = $node->safe_psql( + 'postgres', + 'INSERT INTO test_enc1 (k) VALUES (5),(6);', + extra_params => ['-a']); +PGTDE::append_to_result_file($stdout); + +$stdout = $node->safe_psql( + 'postgres', + 'SELECT * FROM test_enc1 ORDER BY id ASC;', + extra_params => ['-a']); +PGTDE::append_to_result_file($stdout); + +PGTDE::append_to_result_file("-- server restart"); +$node->restart; + +$stdout = $node->safe_psql( + 'postgres', + 'SELECT * FROM test_enc1 ORDER BY id ASC;', + extra_params => ['-a']); +PGTDE::append_to_result_file($stdout); + +$stdout = $node->safe_psql( + 'postgres', + 'DROP TABLE test_enc1;', + extra_params => ['-a']); +PGTDE::append_to_result_file($stdout); + +# Print PGSM settings +($cmdret, $stdout, $stderr) = $node->psql( + 'postgres', + "SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_query_shared_buffer';", + extra_params => [ '-a', '-Pformat=aligned', '-Ptuples_only=off' ]); ok($cmdret == 0, "Print PGTDE EXTENSION Settings"); PGTDE::append_to_debug_file($stdout); # Create example database and run pgbench init -($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE database example;', extra_params => ['-a']); +($cmdret, $stdout, $stderr) = + $node->psql('postgres', 'CREATE database example;', extra_params => ['-a']); print "cmdret $cmdret\n"; ok($cmdret == 0, "CREATE Database example"); PGTDE::append_to_debug_file($stdout); @@ -123,36 +177,42 @@ my $port = $node->port; print "port $port \n"; -my $out = system ("pgbench -i -s 20 -p $port example"); +my $out = system("pgbench -i -s 20 -p $port example"); print " out: $out \n"; ok($cmdret == 0, "Perform pgbench init"); -$out = system ("pgbench -c 10 -j 2 -t 5000 -p $port example"); +$out = system("pgbench -c 10 -j 2 -t 5000 -p $port example"); print " out: $out \n"; ok($cmdret == 0, "Run pgbench"); -($cmdret, $stdout, $stderr) = $node->psql('postgres', 'SELECT datname, substr(query,0,150) AS query, SUM(calls) AS calls FROM pg_stat_monitor GROUP BY datname, query ORDER BY datname, query, calls;', extra_params => ['-a', '-Pformat=aligned','-Ptuples_only=off']); +($cmdret, $stdout, $stderr) = $node->psql( + 'postgres', + 'SELECT datname, substr(query,0,150) AS query, SUM(calls) AS calls FROM pg_stat_monitor GROUP BY datname, query ORDER BY datname, query, calls;', + extra_params => [ '-a', '-Pformat=aligned', '-Ptuples_only=off' ]); ok($cmdret == 0, "SELECT XXX FROM pg_stat_monitor"); PGTDE::append_to_debug_file($stdout); -# DROP EXTENSION -$stdout = $node->safe_psql('postgres', 'DROP EXTENSION pg_tde;', extra_params => ['-a']); +$stdout = $node->safe_psql( + 'postgres', + 'DROP EXTENSION pg_tde;', + extra_params => ['-a']); ok($cmdret == 0, "DROP PGTDE EXTENSION"); -PGTDE::append_to_file($stdout); +PGTDE::append_to_result_file($stdout); -# DROP EXTENSION -$stdout = $node->safe_psql('postgres', 'DROP EXTENSION pg_stat_monitor;', extra_params => ['-a']); +$stdout = $node->safe_psql( + 'postgres', + 'DROP EXTENSION pg_stat_monitor;', + extra_params => ['-a']); ok($cmdret == 0, "DROP PGTDE EXTENSION"); PGTDE::append_to_debug_file($stdout); -# Stop the server -$node->stop(); +$node->stop; -# compare the expected and out file +# Compare the expected and out file my $compare = PGTDE->compare_results(); -# Test/check if expected and result/out file match. If Yes, test passes. -is($compare,0,"Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files."); +is($compare, 0, + "Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files." +); -# Done testing for this testcase file. done_testing(); diff --git a/contrib/pg_tde/t/006_remote_vault_config.pl b/contrib/pg_tde/t/006_remote_vault_config.pl index 336f4fd67691b..1012d9168dcce 100644 --- a/contrib/pg_tde/t/006_remote_vault_config.pl +++ b/contrib/pg_tde/t/006_remote_vault_config.pl @@ -2,119 +2,105 @@ use strict; use warnings; +use Env; use File::Basename; -use File::Compare; -use File::Copy; use Test::More; use lib 't'; use pgtde; -use Env; - -# Get file name and CREATE out file name and dirs WHERE requried -PGTDE::setup_files_dir(basename($0)); - -# CREATE new PostgreSQL node and do initdb -my $node = PGTDE->pgtde_init_pg(); -my $pgdata = $node->data_dir; { -package MyWebServer; - -use HTTP::Server::Simple::CGI; -use base qw(HTTP::Server::Simple::CGI); - -my %dispatch = ( - '/token' => \&resp_token, - '/url' => \&resp_url, - # ... -); - -sub handle_request { - my $self = shift; - my $cgi = shift; - - my $path = $cgi->path_info(); - my $handler = $dispatch{$path}; - - if (ref($handler) eq "CODE") { - print "HTTP/1.0 200 OK\r\n"; - $handler->($cgi); - - } else { - print "HTTP/1.0 404 Not found\r\n"; - print $cgi->header, - $cgi->start_html('Not found'), - $cgi->h1('Not found'), - $cgi->end_html; - } -} -sub resp_token { - my $cgi = shift; - print $cgi->header, - "$ENV{'ROOT_TOKEN'}\r\n"; -} + package MyWebServer; + + use HTTP::Server::Simple::CGI; + use base qw(HTTP::Server::Simple::CGI); + + my %dispatch = ( + '/token' => \&resp_token, + '/url' => \&resp_url,); + + sub handle_request + { + my $self = shift; + my $cgi = shift; + + my $path = $cgi->path_info(); + my $handler = $dispatch{$path}; + + if (ref($handler) eq "CODE") + { + print "HTTP/1.0 200 OK\r\n"; + $handler->($cgi); + + } + else + { + print "HTTP/1.0 404 Not found\r\n"; + print $cgi->header, + $cgi->start_html('Not found'), + $cgi->h1('Not found'), + $cgi->end_html; + } + } + + sub resp_token + { + my $cgi = shift; + print $cgi->header, "$ENV{'ROOT_TOKEN'}\r\n"; + } + + sub resp_url + { + my $cgi = shift; + print $cgi->header, "http://127.0.0.1:8200\r\n"; + } -sub resp_url { - my $cgi = shift; - print $cgi->header, - "http://127.0.0.1:8200\r\n"; -} - } + my $pid = MyWebServer->new(8889)->background(); +PGTDE::setup_files_dir(basename($0)); -# UPDATE postgresql.conf to include/load pg_tde library -open my $conf, '>>', "$pgdata/postgresql.conf"; -print $conf "shared_preload_libraries = 'pg_tde'\n"; -close $conf; +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf('postgresql.conf', "shared_preload_libraries = 'pg_tde'"); +$node->start; -my $rt_value = $node->stop(); -$rt_value = $node->start(); -ok($rt_value == 1, "Restart Server"); +PGTDE::psql($node, 'postgres', 'CREATE EXTENSION IF NOT EXISTS pg_tde;'); -my ($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS pg_tde;', extra_params => ['-a']); -ok($cmdret == 0, "CREATE PGTDE EXTENSION"); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_add_database_key_provider_vault_v2('vault-provider', json_object( 'type' VALUE 'remote', 'url' VALUE 'http://localhost:8889/token' ), json_object( 'type' VALUE 'remote', 'url' VALUE 'http://localhost:8889/url' ), to_json('secret'::text), NULL);" +); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_key_using_database_key_provider('test-db-key','vault-provider');" +); -$rt_value = $node->psql('postgres', "SELECT pg_tde_add_key_provider_vault_v2('vault-provider', json_object( 'type' VALUE 'remote', 'url' VALUE 'http://localhost:8889/token' ), json_object( 'type' VALUE 'remote', 'url' VALUE 'http://localhost:8889/url' ), to_json('secret'::text), NULL);", extra_params => ['-a']); -$rt_value = $node->psql('postgres', "SELECT pg_tde_set_principal_key('test-db-principal-key','vault-provider');", extra_params => ['-a']); +PGTDE::psql($node, 'postgres', + 'CREATE TABLE test_enc2(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap;' +); -$stdout = $node->safe_psql('postgres', 'CREATE TABLE test_enc2(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'INSERT INTO test_enc2 (k) VALUES (5),(6);'); -$stdout = $node->safe_psql('postgres', 'INSERT INTO test_enc2 (k) VALUES (5),(6);', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc2 ORDER BY id ASC;'); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc2 ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::append_to_result_file("-- server restart"); +$node->restart; -# Restart the server -PGTDE::append_to_file("-- server restart"); -$rt_value = $node->stop(); -$rt_value = $node->start(); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc2 ORDER BY id ASC;'); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc2 ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'DROP TABLE test_enc2;'); -$stdout = $node->safe_psql('postgres', 'DROP TABLE test_enc2;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'DROP EXTENSION pg_tde;'); -# DROP EXTENSION -$stdout = $node->safe_psql('postgres', 'DROP EXTENSION pg_tde;', extra_params => ['-a']); -ok($cmdret == 0, "DROP PGTDE EXTENSION"); -PGTDE::append_to_file($stdout); -# Stop the server -$node->stop(); +$node->stop; system("kill -9 $pid"); -# compare the expected and out file +# Compare the expected and out file my $compare = PGTDE->compare_results(); -# Test/check if expected and result/out file match. If Yes, test passes. -is($compare,0,"Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files."); +is($compare, 0, + "Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files." +); -# Done testing for this testcase file. done_testing(); diff --git a/contrib/pg_tde/t/007_tde_heap.pl b/contrib/pg_tde/t/007_tde_heap.pl index 77b7098bcbb47..09e72806daa3c 100644 --- a/contrib/pg_tde/t/007_tde_heap.pl +++ b/contrib/pg_tde/t/007_tde_heap.pl @@ -3,151 +3,125 @@ use strict; use warnings; use File::Basename; -use File::Compare; -use File::Copy; use Test::More; use lib 't'; use pgtde; -# Get file name and CREATE out file name and dirs WHERE requried PGTDE::setup_files_dir(basename($0)); -my $PG_VERSION_STRING = `pg_config --version`; +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf('postgresql.conf', "shared_preload_libraries = 'pg_tde'"); +$node->start; -if (index(lc($PG_VERSION_STRING), lc("Percona Server")) == -1) -{ - plan skip_all => "pg_tde test case only for Percona Server for PostgreSQL"; -} - -# CREATE new PostgreSQL node and do initdb -my $node = PGTDE->pgtde_init_pg(); -my $pgdata = $node->data_dir; - -# UPDATE postgresql.conf to include/load pg_tde library -open my $conf, '>>', "$pgdata/postgresql.conf"; -print $conf "shared_preload_libraries = 'pg_tde'\n"; -close $conf; - -# Start server -my $rt_value = $node->start; -ok($rt_value == 1, "Start Server"); - -# CREATE EXTENSION IF NOT EXISTS and change out file permissions -my ($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS pg_tde;', extra_params => ['-a']); -ok($cmdret == 0, "CREATE PGTDE EXTENSION"); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'CREATE EXTENSION IF NOT EXISTS pg_tde;'); +PGTDE::psql($node, 'postgres', + 'CREATE TABLE test_enc(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap;' +); -$rt_value = $node->psql('postgres', 'CREATE TABLE test_enc(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap;', extra_params => ['-a']); -ok($rt_value == 3, "Failing query"); - - -# Restart the server -PGTDE::append_to_file("-- server restart"); -$node->stop(); - -$rt_value = $node->start(); -ok($rt_value == 1, "Restart Server"); - -$rt_value = $node->psql('postgres', "SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');", extra_params => ['-a']); -$rt_value = $node->psql('postgres', "SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault');", extra_params => ['-a']); - +PGTDE::append_to_result_file("-- server restart"); +$node->restart; +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');" +); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault');" +); ######################### test_enc1 (simple create table w tde_heap) +PGTDE::psql($node, 'postgres', + 'CREATE TABLE test_enc1(id SERIAL,k VARCHAR(32),PRIMARY KEY (id)) USING tde_heap;' +); -$stdout = $node->safe_psql('postgres', 'CREATE TABLE test_enc1(id SERIAL,k VARCHAR(32),PRIMARY KEY (id)) USING tde_heap;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -$stdout = $node->safe_psql('postgres', 'INSERT INTO test_enc1 (k) VALUES (\'foobar\'),(\'barfoo\');', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', + 'INSERT INTO test_enc1 (k) VALUES (\'foobar\'),(\'barfoo\');'); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc1 ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc1 ORDER BY id ASC;'); ######################### test_enc2 (create heap + alter to tde_heap) -$stdout = $node->safe_psql('postgres', 'CREATE TABLE test_enc2(id SERIAL,k VARCHAR(32),PRIMARY KEY (id));', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', + 'CREATE TABLE test_enc2(id SERIAL,k VARCHAR(32),PRIMARY KEY (id));'); -$stdout = $node->safe_psql('postgres', 'INSERT INTO test_enc2 (k) VALUES (\'foobar\'),(\'barfoo\');', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', + 'INSERT INTO test_enc2 (k) VALUES (\'foobar\'),(\'barfoo\');'); -$stdout = $node->safe_psql('postgres', 'ALTER TABLE test_enc2 SET ACCESS METHOD tde_heap;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', + 'ALTER TABLE test_enc2 SET ACCESS METHOD tde_heap;'); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc2 ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc2 ORDER BY id ASC;'); ######################### test_enc3 (default_table_access_method) -$stdout = $node->safe_psql('postgres', 'SET default_table_access_method = "tde_heap"; CREATE TABLE test_enc3(id SERIAL,k VARCHAR(32),PRIMARY KEY (id));', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', + 'SET default_table_access_method = "tde_heap"; CREATE TABLE test_enc3(id SERIAL,k VARCHAR(32),PRIMARY KEY (id));' +); -$stdout = $node->safe_psql('postgres', 'INSERT INTO test_enc3 (k) VALUES (\'foobar\'),(\'barfoo\');', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', + 'INSERT INTO test_enc3 (k) VALUES (\'foobar\'),(\'barfoo\');'); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc3 ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc3 ORDER BY id ASC;'); ######################### test_enc4 (create heap + alter default) -$stdout = $node->safe_psql('postgres', 'CREATE TABLE test_enc4(id SERIAL,k VARCHAR(32),PRIMARY KEY (id)) USING heap;', extra_params => ['-a']); +PGTDE::psql($node, 'postgres', + 'CREATE TABLE test_enc4(id SERIAL,k VARCHAR(32),PRIMARY KEY (id)) USING heap;' +); -$stdout = $node->safe_psql('postgres', 'INSERT INTO test_enc4 (k) VALUES (\'foobar\'),(\'barfoo\');', extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', 'SET default_table_access_method = "tde_heap"; ALTER TABLE test_enc4 SET ACCESS METHOD DEFAULT;', extra_params => ['-a']); +PGTDE::psql($node, 'postgres', + 'INSERT INTO test_enc4 (k) VALUES (\'foobar\'),(\'barfoo\');'); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc4 ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', + 'SET default_table_access_method = "tde_heap"; ALTER TABLE test_enc4 SET ACCESS METHOD DEFAULT;' +); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc4 ORDER BY id ASC;'); ######################### test_enc5 (create tde_heap + truncate) -$stdout = $node->safe_psql('postgres', 'CREATE TABLE test_enc5(id SERIAL,k VARCHAR(32),PRIMARY KEY (id)) USING tde_heap;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', + 'CREATE TABLE test_enc5(id SERIAL,k VARCHAR(32),PRIMARY KEY (id)) USING tde_heap;' +); -$stdout = $node->safe_psql('postgres', 'INSERT INTO test_enc5 (k) VALUES (\'foobar\'),(\'barfoo\');', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', + 'INSERT INTO test_enc5 (k) VALUES (\'foobar\'),(\'barfoo\');'); -$stdout = $node->safe_psql('postgres', 'CHECKPOINT;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'CHECKPOINT;'); -$stdout = $node->safe_psql('postgres', 'TRUNCATE test_enc5;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'TRUNCATE test_enc5;'); -$stdout = $node->safe_psql('postgres', 'INSERT INTO test_enc5 (k) VALUES (\'foobar\'),(\'barfoo\');', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', + 'INSERT INTO test_enc5 (k) VALUES (\'foobar\'),(\'barfoo\');'); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc5 ORDER BY id ASC;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc5 ORDER BY id ASC;'); -# Restart the server -PGTDE::append_to_file("-- server restart"); -$rt_value = $node->stop(); -$rt_value = $node->start(); +PGTDE::append_to_result_file("-- server restart"); +$node->restart; sub verify_table { - PGTDE::append_to_file('###########################'); + PGTDE::append_to_result_file('###########################'); - my ($table) = @_; + my ($table) = @_; - my $tablefile = $node->safe_psql('postgres', 'SHOW data_directory;'); - $tablefile .= '/'; - $tablefile .= $node->safe_psql('postgres', 'SELECT pg_relation_filepath(\''.$table.'\');'); + my $tablefile = $node->safe_psql('postgres', 'SHOW data_directory;'); + $tablefile .= '/'; + $tablefile .= $node->safe_psql('postgres', + 'SELECT pg_relation_filepath(\'' . $table . '\');'); - $stdout = $node->safe_psql('postgres', 'SELECT * FROM ' . $table . ' ORDER BY id ASC;', extra_params => ['-a']); - PGTDE::append_to_file($stdout); + PGTDE::psql($node, 'postgres', + 'SELECT * FROM ' . $table . ' ORDER BY id ASC;'); - my $strings = 'TABLEFILE FOR ' . $table . ' FOUND: '; - $strings .= `(ls $tablefile >/dev/null && echo -n yes) || echo -n no`; - PGTDE::append_to_file($strings); + my $strings = 'TABLEFILE FOR ' . $table . ' FOUND: '; + $strings .= `(ls $tablefile >/dev/null && echo -n yes) || echo -n no`; + PGTDE::append_to_result_file($strings); - $strings = 'CONTAINS FOO (should be empty): '; - $strings .= `strings $tablefile | grep foo`; - PGTDE::append_to_file($strings); + $strings = 'CONTAINS FOO (should be empty): '; + $strings .= `strings $tablefile | grep foo`; + PGTDE::append_to_result_file($strings); } verify_table('test_enc1'); @@ -159,77 +133,60 @@ sub verify_table # Verify that we can't see the data in the file my $tablefile2 = $node->safe_psql('postgres', 'SHOW data_directory;'); $tablefile2 .= '/'; -$tablefile2 .= $node->safe_psql('postgres', 'SELECT pg_relation_filepath(\'test_enc2\');'); +$tablefile2 .= + $node->safe_psql('postgres', 'SELECT pg_relation_filepath(\'test_enc2\');'); -my $strings = 'TABLEFILE2 FOUND: '; +my $strings = 'TABLEFILE2 FOUND: '; $strings .= `(ls $tablefile2 >/dev/null && echo yes) || echo no`; -PGTDE::append_to_file($strings); +PGTDE::append_to_result_file($strings); $strings = 'CONTAINS FOO (should be empty): '; $strings .= `strings $tablefile2 | grep foo`; -PGTDE::append_to_file($strings); - - - +PGTDE::append_to_result_file($strings); # Verify that we can't see the data in the file my $tablefile3 = $node->safe_psql('postgres', 'SHOW data_directory;'); $tablefile3 .= '/'; -$tablefile3 .= $node->safe_psql('postgres', 'SELECT pg_relation_filepath(\'test_enc3\');'); +$tablefile3 .= + $node->safe_psql('postgres', 'SELECT pg_relation_filepath(\'test_enc3\');'); $strings = 'TABLEFILE3 FOUND: '; $strings .= `(ls $tablefile3 >/dev/null && echo yes) || echo no`; -PGTDE::append_to_file($strings); +PGTDE::append_to_result_file($strings); $strings = 'CONTAINS FOO (should be empty): '; $strings .= `strings $tablefile3 | grep foo`; -PGTDE::append_to_file($strings); - - - +PGTDE::append_to_result_file($strings); # Verify that we can't see the data in the file my $tablefile4 = $node->safe_psql('postgres', 'SHOW data_directory;'); $tablefile4 .= '/'; -$tablefile4 .= $node->safe_psql('postgres', 'SELECT pg_relation_filepath(\'test_enc4\');'); +$tablefile4 .= + $node->safe_psql('postgres', 'SELECT pg_relation_filepath(\'test_enc4\');'); $strings = 'TABLEFILE4 FOUND: '; $strings .= `(ls $tablefile4 >/dev/null && echo yes) || echo no`; -PGTDE::append_to_file($strings); +PGTDE::append_to_result_file($strings); $strings = 'CONTAINS FOO (should be empty): '; $strings .= `strings $tablefile4 | grep foo`; -PGTDE::append_to_file($strings); - - - -$stdout = $node->safe_psql('postgres', 'DROP TABLE test_enc1;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -$stdout = $node->safe_psql('postgres', 'DROP TABLE test_enc2;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -$stdout = $node->safe_psql('postgres', 'DROP TABLE test_enc3;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::append_to_result_file($strings); -$stdout = $node->safe_psql('postgres', 'DROP TABLE test_enc4;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'DROP TABLE test_enc1;'); +PGTDE::psql($node, 'postgres', 'DROP TABLE test_enc2;'); +PGTDE::psql($node, 'postgres', 'DROP TABLE test_enc3;'); +PGTDE::psql($node, 'postgres', 'DROP TABLE test_enc4;'); +PGTDE::psql($node, 'postgres', 'DROP TABLE test_enc5;'); -$stdout = $node->safe_psql('postgres', 'DROP TABLE test_enc5;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'DROP EXTENSION pg_tde;'); -# DROP EXTENSION -$stdout = $node->safe_psql('postgres', 'DROP EXTENSION pg_tde;', extra_params => ['-a']); -ok($cmdret == 0, "DROP PGTDE EXTENSION"); -PGTDE::append_to_file($stdout); -# Stop the server -$node->stop(); +$node->stop; -# compare the expected and out file +# Compare the expected and out file my $compare = PGTDE->compare_results(); -# Test/check if expected and result/out file match. If Yes, test passes. -is($compare,0,"Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files."); +is($compare, 0, + "Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files." +); -# Done testing for this testcase file. done_testing(); diff --git a/contrib/pg_tde/t/008_key_rotate_tablespace.pl b/contrib/pg_tde/t/008_key_rotate_tablespace.pl index c540276985b6a..9369940d9e8a9 100644 --- a/contrib/pg_tde/t/008_key_rotate_tablespace.pl +++ b/contrib/pg_tde/t/008_key_rotate_tablespace.pl @@ -3,94 +3,71 @@ use strict; use warnings; use File::Basename; -use File::Compare; -use File::Copy; use Test::More; use lib 't'; use pgtde; -# Get file name and CREATE out file name and dirs WHERE requried PGTDE::setup_files_dir(basename($0)); -my ($cmdret, $stdout); - -# CREATE new PostgreSQL node and do initdb -my $node = PGTDE->pgtde_init_pg(); -my $pgdata = $node->data_dir; - -# UPDATE postgresql.conf to include/load pg_tde library -open my $conf, '>>', "$pgdata/postgresql.conf"; -print $conf "shared_preload_libraries = 'pg_tde'\n"; -close $conf; - -# Start server -my $rt_value = $node->start; -ok($rt_value == 1, "Start Server"); - -$node->safe_psql('postgres', - q{ -SET allow_in_place_tablespaces = true; -CREATE TABLESPACE test_tblspace LOCATION ''; -CREATE DATABASE tbc TABLESPACE = test_tblspace; -}); - -$stdout = $node->safe_psql('tbc', - q{ -CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); - +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf('postgresql.conf', "shared_preload_libraries = 'pg_tde'"); +$node->start; + +PGTDE::psql($node, 'postgres', + "SET allow_in_place_tablespaces = true; CREATE TABLESPACE test_tblspace LOCATION '';" +); +PGTDE::psql($node, 'postgres', + 'CREATE DATABASE tbc TABLESPACE = test_tblspace;'); + +PGTDE::psql($node, 'tbc', 'CREATE EXTENSION IF NOT EXISTS pg_tde;'); +PGTDE::psql($node, 'tbc', + "SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');" +); +PGTDE::psql($node, 'tbc', + "SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault');" +); + +PGTDE::psql( + $node, 'tbc', " CREATE TABLE country_table ( country_id serial primary key, country_name text unique not null, continent text not null ) USING tde_heap; +"); +PGTDE::psql( + $node, 'tbc', " INSERT INTO country_table (country_name, continent) VALUES ('Japan', 'Asia'), ('UK', 'Europe'), ('USA', 'North America'); +"); -SELECT * FROM country_table; - -}, extra_params => ['-a']); -PGTDE::append_to_file($stdout); - - -$cmdret = $node->psql('tbc', "SELECT pg_tde_set_principal_key('new-k', 'file-vault');", extra_params => ['-a']); -ok($cmdret == 0, "ROTATE KEY"); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'tbc', 'SELECT * FROM country_table;'); -# Restart the server -PGTDE::append_to_file("-- server restart"); -$node->stop(); +PGTDE::psql($node, 'tbc', + "SELECT pg_tde_set_key_using_database_key_provider('new-k', 'file-vault');" +); -$rt_value = $node->start(); -ok($rt_value == 1, "Restart Server"); +PGTDE::append_to_result_file("-- server restart"); +$node->restart; -$stdout = $node->safe_psql('tbc', 'SELECT * FROM country_table;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'tbc', 'SELECT * FROM country_table;'); +PGTDE::psql($node, 'tbc', 'DROP EXTENSION pg_tde CASCADE;'); -# DROP EXTENSION -$stdout = $node->safe_psql('tbc', 'DROP EXTENSION pg_tde CASCADE;', extra_params => ['-a']); -ok($cmdret == 0, "DROP PGTDE EXTENSION"); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'DROP DATABASE tbc;'); +PGTDE::psql($node, 'postgres', 'DROP TABLESPACE test_tblspace;'); -$stdout = $node->safe_psql('postgres', q{ -DROP DATABASE tbc; -DROP TABLESPACE test_tblspace; -}, extra_params => ['-a']); -ok($cmdret == 0, "DROP DATABSE"); -PGTDE::append_to_file($stdout); -# Stop the server -$node->stop(); +$node->stop; -# compare the expected and out file +# Compare the expected and out file my $compare = PGTDE->compare_results(); -# Test/check if expected and result/out file match. If Yes, test passes. -is($compare,0,"Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files."); +is($compare, 0, + "Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files." +); -# Done testing for this testcase file. done_testing(); diff --git a/contrib/pg_tde/t/009_wal_encrypt.pl b/contrib/pg_tde/t/009_wal_encrypt.pl index 77e9633ca41f5..e4ec0b7dc888d 100644 --- a/contrib/pg_tde/t/009_wal_encrypt.pl +++ b/contrib/pg_tde/t/009_wal_encrypt.pl @@ -3,71 +3,86 @@ use strict; use warnings; use File::Basename; -use File::Compare; -use File::Copy; use Test::More; use lib 't'; use pgtde; -# Get file name and CREATE out file name and dirs WHERE requried PGTDE::setup_files_dir(basename($0)); -# CREATE new PostgreSQL node and do initdb -my $node = PGTDE->pgtde_init_pg(); -my $pgdata = $node->data_dir; - -# UPDATE postgresql.conf to include/load pg_tde library -open my $conf, '>>', "$pgdata/postgresql.conf"; -print $conf "shared_preload_libraries = 'pg_tde'\n"; +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf('postgresql.conf', "shared_preload_libraries = 'pg_tde'"); +$node->append_conf('postgresql.conf', "wal_level = 'logical'"); # NOT testing that it can't start: the test framework doesn't have an easy way to do this -# print $conf "pg_tde.wal_encrypt = 1\n"; -close $conf; +#$node->append_conf('postgresql.conf', "pg_tde.wal_encrypt = 1"}); +$node->start; + +PGTDE::psql($node, 'postgres', "CREATE EXTENSION IF NOT EXISTS pg_tde;"); + +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_add_global_key_provider_file('file-keyring-010','/tmp/pg_tde_test_keyring010.per');" +); + +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_server_key_using_global_key_provider('server-key', 'file-keyring-010');" +); + +PGTDE::psql($node, 'postgres', 'ALTER SYSTEM SET pg_tde.wal_encrypt = on;'); + +PGTDE::append_to_result_file("-- server restart with wal encryption"); +$node->restart; + +PGTDE::psql($node, 'postgres', "SHOW pg_tde.wal_encrypt;"); + +PGTDE::psql($node, 'postgres', + "SELECT slot_name FROM pg_create_logical_replication_slot('tde_slot', 'test_decoding');" +); + +PGTDE::psql($node, 'postgres', + 'CREATE TABLE test_wal (id SERIAL, k INTEGER, PRIMARY KEY (id));'); + +PGTDE::psql($node, 'postgres', 'INSERT INTO test_wal (k) VALUES (1), (2);'); + +PGTDE::psql($node, 'postgres', 'ALTER SYSTEM SET pg_tde.wal_encrypt = off;'); + +PGTDE::append_to_result_file("-- server restart without wal encryption"); +$node->restart; + +PGTDE::psql($node, 'postgres', "SHOW pg_tde.wal_encrypt;"); -my $rt_value = $node->start; -ok($rt_value == 1, "Start Server"); +PGTDE::psql($node, 'postgres', 'INSERT INTO test_wal (k) VALUES (3), (4);'); -my $stdout = $node->safe_psql('postgres', "CREATE EXTENSION IF NOT EXISTS pg_tde;", extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'ALTER SYSTEM SET pg_tde.wal_encrypt = on;'); -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_add_global_key_provider_file('file-keyring-010','/tmp/pg_tde_test_keyring010.per');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::append_to_result_file("-- server restart with wal encryption"); +$node->restart; -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_set_server_principal_key('global-db-principal-key', 'file-keyring-010');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', "SHOW pg_tde.wal_encrypt;"); -# Restart the server, it should work with encryption now -PGTDE::append_to_file("-- server restart with wal encryption"); -$node->stop(); +PGTDE::psql($node, 'postgres', 'INSERT INTO test_wal (k) VALUES (5), (6);'); -open $conf, '>>', "$pgdata/postgresql.conf"; -print $conf "pg_tde.wal_encrypt = 1\n"; -close $conf; +PGTDE::append_to_result_file("-- server restart with still wal encryption"); +$node->restart; -$rt_value = $node->start(); -ok($rt_value == 1, "Restart Server"); +PGTDE::psql($node, 'postgres', "SHOW pg_tde.wal_encrypt;"); -$stdout = $node->safe_psql('postgres', "SHOW pg_tde.wal_encrypt;", extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', 'INSERT INTO test_wal (k) VALUES (7), (8);'); -$stdout = $node->safe_psql('postgres', 'CREATE TABLE test_wal(id SERIAL,k INTEGER,PRIMARY KEY (id));', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', + "SELECT data FROM pg_logical_slot_get_changes('tde_slot', NULL, NULL);"); -$stdout = $node->safe_psql('postgres', 'CHECKPOINT;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', + "SELECT pg_drop_replication_slot('tde_slot');"); -# TODO: add WAL content testing after the wal rework +PGTDE::psql($node, 'postgres', 'DROP EXTENSION pg_tde;'); -# DROP EXTENSION -$stdout = $node->safe_psql('postgres', 'DROP EXTENSION pg_tde;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); -# Stop the server -$node->stop(); +$node->stop; -# compare the expected and out file +# Compare the expected and out file my $compare = PGTDE->compare_results(); -# Test/check if expected and result/out file match. If Yes, test passes. -is($compare,0,"Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files."); +is($compare, 0, + "Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files." +); -# Done testing for this testcase file. done_testing(); diff --git a/contrib/pg_tde/t/010_change_key_provider.pl b/contrib/pg_tde/t/010_change_key_provider.pl index c7add5bb551bf..60b40243222bf 100644 --- a/contrib/pg_tde/t/010_change_key_provider.pl +++ b/contrib/pg_tde/t/010_change_key_provider.pl @@ -3,207 +3,150 @@ use strict; use warnings; use File::Basename; -use File::Compare; use File::Copy; use Test::More; use lib 't'; use pgtde; -# Get file name and CREATE out file name and dirs WHERE requried PGTDE::setup_files_dir(basename($0)); -# CREATE new PostgreSQL node and do initdb -my $node = PGTDE->pgtde_init_pg(); -my $pgdata = $node->data_dir; - -# UPDATE postgresql.conf to include/load pg_tde library -open my $conf, '>>', "$pgdata/postgresql.conf"; -print $conf "shared_preload_libraries = 'pg_tde'\n"; -close $conf; - unlink('/tmp/change_key_provider_1.per'); unlink('/tmp/change_key_provider_2.per'); unlink('/tmp/change_key_provider_3.per'); unlink('/tmp/change_key_provider_4.per'); -# Start server -my $rt_value = $node->start; -ok($rt_value == 1, "Start Server"); - -# CREATE EXTENSION IF NOT EXISTS and change out file permissions -my ($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS pg_tde;', extra_params => ['-a']); -ok($cmdret == 0, "CREATE PGTDE EXTENSION"); -PGTDE::append_to_file($stdout); - -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_add_key_provider_file('file-vault', '/tmp/change_key_provider_1.per');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_list_all_key_providers();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_set_principal_key('test-key', 'file-vault');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -$stdout = $node->safe_psql('postgres', 'CREATE TABLE test_enc (id serial, k integer, PRIMARY KEY (id)) USING tde_heap;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', 'INSERT INTO test_enc (k) VALUES (5), (6);', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_verify_principal_key();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_is_encrypted('test_enc');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc ORDER BY id;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf('postgresql.conf', "shared_preload_libraries = 'pg_tde'"); +$node->start; + +PGTDE::psql($node, 'postgres', 'CREATE EXTENSION IF NOT EXISTS pg_tde;'); + +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_add_database_key_provider_file('file-vault', '/tmp/change_key_provider_1.per');" +); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_list_all_database_key_providers();"); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_key_using_database_key_provider('test-key', 'file-vault');" +); + +PGTDE::psql($node, 'postgres', + 'CREATE TABLE test_enc (id serial, k integer, PRIMARY KEY (id)) USING tde_heap;' +); +PGTDE::psql($node, 'postgres', 'INSERT INTO test_enc (k) VALUES (5), (6);'); + +PGTDE::psql($node, 'postgres', "SELECT pg_tde_verify_key();"); +PGTDE::psql($node, 'postgres', "SELECT pg_tde_is_encrypted('test_enc');"); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc ORDER BY id;'); # Change provider and move file -PGTDE::append_to_file("-- mv /tmp/change_key_provider_1.per /tmp/change_key_provider_2.per"); +PGTDE::append_to_result_file( + "-- mv /tmp/change_key_provider_1.per /tmp/change_key_provider_2.per"); move('/tmp/change_key_provider_1.per', '/tmp/change_key_provider_2.per'); -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_change_key_provider_file('file-vault', '/tmp/change_key_provider_2.per');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_list_all_key_providers();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_verify_principal_key();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_is_encrypted('test_enc');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc ORDER BY id;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -# Restart the server -PGTDE::append_to_file("-- server restart"); -$rt_value = $node->stop(); -$rt_value = $node->start(); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_change_database_key_provider_file('file-vault', '/tmp/change_key_provider_2.per');" +); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_list_all_database_key_providers();"); + +PGTDE::psql($node, 'postgres', "SELECT pg_tde_verify_key();"); +PGTDE::psql($node, 'postgres', "SELECT pg_tde_is_encrypted('test_enc');"); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc ORDER BY id;'); + +PGTDE::append_to_result_file("-- server restart"); +$node->restart; # Verify -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_verify_principal_key();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_is_encrypted('test_enc');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc ORDER BY id;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', "SELECT pg_tde_verify_key();"); +PGTDE::psql($node, 'postgres', "SELECT pg_tde_is_encrypted('test_enc');"); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc ORDER BY id;'); # Change provider and do not move file -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_change_key_provider_file('file-vault', '/tmp/change_key_provider_3.per');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_list_all_key_providers();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -(undef, $stdout, $stderr) = $node->psql('postgres', "SELECT pg_tde_verify_principal_key();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -PGTDE::append_to_file($stderr); -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_is_encrypted('test_enc');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc ORDER BY id;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -# Restart the server -PGTDE::append_to_file("-- server restart"); -$rt_value = $node->stop(); -$rt_value = $node->start(); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_change_database_key_provider_file('file-vault', '/tmp/change_key_provider_3.per');" +); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_list_all_database_key_providers();"); + +PGTDE::psql($node, 'postgres', "SELECT pg_tde_verify_key();"); +PGTDE::psql($node, 'postgres', "SELECT pg_tde_is_encrypted('test_enc');"); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc ORDER BY id;'); + +PGTDE::append_to_result_file("-- server restart"); +$node->restart; # Verify -(undef, $stdout, $stderr) = $node->psql('postgres', "SELECT pg_tde_verify_principal_key();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -PGTDE::append_to_file($stderr); -(undef, $stdout, $stderr) = $node->psql('postgres', "SELECT pg_tde_is_encrypted('test_enc');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -PGTDE::append_to_file($stderr); -(undef, $stdout, $stderr) = $node->psql('postgres', 'SELECT * FROM test_enc ORDER BY id;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); -PGTDE::append_to_file($stderr); - -PGTDE::append_to_file("-- mv /tmp/change_key_provider_2.per /tmp/change_key_provider_3.per"); +PGTDE::psql($node, 'postgres', "SELECT pg_tde_verify_key();"); +PGTDE::psql($node, 'postgres', "SELECT pg_tde_is_encrypted('test_enc');"); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc ORDER BY id;'); + +PGTDE::append_to_result_file( + "-- mv /tmp/change_key_provider_2.per /tmp/change_key_provider_3.per"); move('/tmp/change_key_provider_2.per', '/tmp/change_key_provider_3.per'); -# Restart the server -PGTDE::append_to_file("-- server restart"); -$rt_value = $node->stop(); -$rt_value = $node->start(); +PGTDE::append_to_result_file("-- server restart"); +$node->restart; # Verify -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_verify_principal_key();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_is_encrypted('test_enc');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc ORDER BY id;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -# DROP EXTENSION -(undef, $stdout, $stderr) = $node->psql('postgres', 'DROP EXTENSION pg_tde CASCADE;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); -PGTDE::append_to_file($stderr); - -# CREATE EXTENSION -($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS pg_tde;', extra_params => ['-a']); -ok($cmdret == 0, "CREATE PGTDE EXTENSION"); -PGTDE::append_to_file($stdout); +PGTDE::psql($node, 'postgres', "SELECT pg_tde_verify_key();"); +PGTDE::psql($node, 'postgres', "SELECT pg_tde_is_encrypted('test_enc');"); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc ORDER BY id;'); + +PGTDE::psql($node, 'postgres', 'DROP EXTENSION pg_tde CASCADE;'); + +PGTDE::psql($node, 'postgres', 'CREATE EXTENSION IF NOT EXISTS pg_tde;'); # Change provider and generate a new principal key -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_add_key_provider_file('file-vault', '/tmp/change_key_provider_4.per');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->psql('postgres', "SELECT pg_tde_set_principal_key('test-key', 'file-vault');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -$stdout = $node->safe_psql('postgres', 'CREATE TABLE test_enc (id serial, k integer, PRIMARY KEY (id)) USING tde_heap;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', 'INSERT INTO test_enc (k) VALUES (5), (6);', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_verify_principal_key();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_is_encrypted('test_enc');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc ORDER BY id;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_change_key_provider_file('file-vault', '/tmp/change_key_provider_3.per');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -# Restart the server -PGTDE::append_to_file("-- server restart"); -$rt_value = $node->stop(); -$rt_value = $node->start(); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_add_database_key_provider_file('file-vault', '/tmp/change_key_provider_4.per');" +); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_key_using_database_key_provider('test-key', 'file-vault');" +); + +PGTDE::psql($node, 'postgres', + 'CREATE TABLE test_enc (id serial, k integer, PRIMARY KEY (id)) USING tde_heap;' +); +PGTDE::psql($node, 'postgres', 'INSERT INTO test_enc (k) VALUES (5), (6);'); + +PGTDE::psql($node, 'postgres', "SELECT pg_tde_verify_key();"); +PGTDE::psql($node, 'postgres', "SELECT pg_tde_is_encrypted('test_enc');"); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc ORDER BY id;'); + +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_change_database_key_provider_file('file-vault', '/tmp/change_key_provider_3.per');" +); + +PGTDE::append_to_result_file("-- server restart"); +$node->restart; # Verify -(undef, $stdout, $stderr) = $node->psql('postgres', "SELECT pg_tde_verify_principal_key();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -PGTDE::append_to_file($stderr); -(undef, $stdout, $stderr) = $node->psql('postgres', "SELECT pg_tde_is_encrypted('test_enc');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -PGTDE::append_to_file($stderr); -(undef, $stdout, $stderr) = $node->psql('postgres', 'SELECT * FROM test_enc ORDER BY id;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); -PGTDE::append_to_file($stderr); - -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_change_key_provider_file('file-vault', '/tmp/change_key_provider_4.per');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -# Restart the server -PGTDE::append_to_file("-- server restart"); -$rt_value = $node->stop(); -$rt_value = $node->start(); +PGTDE::psql($node, 'postgres', "SELECT pg_tde_verify_key();"); +PGTDE::psql($node, 'postgres', "SELECT pg_tde_is_encrypted('test_enc');"); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc ORDER BY id;'); +PGTDE::psql($node, 'postgres', + 'CREATE TABLE test_enc2 (id serial, k integer, PRIMARY KEY (id)) USING tde_heap;' +); + +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_change_database_key_provider_file('file-vault', '/tmp/change_key_provider_4.per');" +); # Verify -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_verify_principal_key();", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', "SELECT pg_tde_is_encrypted('test_enc');", extra_params => ['-a']); -PGTDE::append_to_file($stdout); -$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc ORDER BY id;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); - -# DROP EXTENSION -(undef, $stdout, $stderr) = $node->psql('postgres', 'DROP EXTENSION pg_tde CASCADE;', extra_params => ['-a']); -PGTDE::append_to_file($stdout); -PGTDE::append_to_file($stderr); - -# Stop the server -$node->stop(); - -# compare the expected and out file +PGTDE::psql($node, 'postgres', "SELECT pg_tde_verify_key();"); +PGTDE::psql($node, 'postgres', "SELECT pg_tde_is_encrypted('test_enc');"); +PGTDE::psql($node, 'postgres', 'SELECT * FROM test_enc ORDER BY id;'); + +PGTDE::psql($node, 'postgres', 'DROP EXTENSION pg_tde CASCADE;'); + +$node->stop; + +# Compare the expected and out file my $compare = PGTDE->compare_results(); -# Test/check if expected and result/out file match. If Yes, test passes. -is($compare, 0, "Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files."); +is($compare, 0, + "Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files." +); -# Done testing for this testcase file. done_testing(); diff --git a/contrib/pg_tde/t/011_unlogged_tables.pl b/contrib/pg_tde/t/011_unlogged_tables.pl new file mode 100644 index 0000000000000..cdf898c54f845 --- /dev/null +++ b/contrib/pg_tde/t/011_unlogged_tables.pl @@ -0,0 +1,51 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use File::Basename; +use Test::More; +use lib 't'; +use pgtde; + +PGTDE::setup_files_dir(basename($0)); + +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf('postgresql.conf', "shared_preload_libraries = 'pg_tde'"); +$node->start; + +PGTDE::psql($node, 'postgres', 'CREATE EXTENSION IF NOT EXISTS pg_tde;'); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_add_database_key_provider_file('file-vault', '/tmp/unlogged_tables.per');" +); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_key_using_database_key_provider('test-key', 'file-vault');" +); + +PGTDE::psql($node, 'postgres', + "CREATE UNLOGGED TABLE t (x int PRIMARY KEY) USING tde_heap;"); + +PGTDE::psql($node, 'postgres', "INSERT INTO t SELECT generate_series(1, 4);"); + +PGTDE::psql($node, 'postgres', "CHECKPOINT;"); + +PGTDE::append_to_result_file("-- kill -9"); +$node->kill9(); + +PGTDE::append_to_result_file("-- server start"); +$node->start; + +PGTDE::psql($node, 'postgres', "TABLE t;"); + +PGTDE::psql($node, 'postgres', "INSERT INTO t SELECT generate_series(1, 4);"); + +$node->stop; + +# Compare the expected and out file +my $compare = PGTDE->compare_results(); + +is($compare, 0, + "Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files." +); + +done_testing(); diff --git a/contrib/pg_tde/t/012_replication.pl b/contrib/pg_tde/t/012_replication.pl new file mode 100644 index 0000000000000..2de29970471f2 --- /dev/null +++ b/contrib/pg_tde/t/012_replication.pl @@ -0,0 +1,69 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use File::Basename; +use Test::More; +use lib 't'; +use pgtde; + +PGTDE::setup_files_dir(basename($0)); + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf('postgresql.conf', + "shared_preload_libraries = 'pg_tde'"); +$primary->start; + +$primary->backup('backup'); +my $replica = PostgreSQL::Test::Cluster->new('replica'); +$replica->init_from_backup($primary, 'backup', has_streaming => 1); +$replica->set_standby_mode(); +$replica->start; + +PGTDE::append_to_result_file("-- At primary"); + +PGTDE::psql($primary, 'postgres', 'CREATE EXTENSION IF NOT EXISTS pg_tde;'); +PGTDE::psql($primary, 'postgres', + "SELECT pg_tde_add_database_key_provider_file('file-vault', '/tmp/unlogged_tables.per');" +); +PGTDE::psql($primary, 'postgres', + "SELECT pg_tde_set_key_using_database_key_provider('test-key', 'file-vault');" +); + +PGTDE::psql($primary, 'postgres', + "CREATE TABLE test_enc (x int PRIMARY KEY) USING tde_heap;"); +PGTDE::psql($primary, 'postgres', + "INSERT INTO test_enc (x) VALUES (1), (2);"); + +PGTDE::psql($primary, 'postgres', + "CREATE TABLE test_plain (x int PRIMARY KEY) USING heap;"); +PGTDE::psql($primary, 'postgres', + "INSERT INTO test_plain (x) VALUES (3), (4);"); + +$primary->wait_for_catchup('replica'); + +PGTDE::append_to_result_file("-- At replica"); + +PGTDE::psql($replica, 'postgres', "SELECT pg_tde_is_encrypted('test_enc');"); +PGTDE::psql($replica, 'postgres', + "SELECT pg_tde_is_encrypted('test_enc_pkey');"); +PGTDE::psql($replica, 'postgres', "SELECT * FROM test_enc ORDER BY x;"); + +PGTDE::psql($replica, 'postgres', + "SELECT pg_tde_is_encrypted('test_plain');"); +PGTDE::psql($replica, 'postgres', + "SELECT pg_tde_is_encrypted('test_plain_pkey');"); +PGTDE::psql($replica, 'postgres', "SELECT * FROM test_plain ORDER BY x;"); + +$replica->stop; +$primary->stop; + +# Compare the expected and out file +my $compare = PGTDE->compare_results(); + +is($compare, 0, + "Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files." +); + +done_testing(); diff --git a/contrib/pg_tde/t/013_crash_recovery.pl b/contrib/pg_tde/t/013_crash_recovery.pl new file mode 100644 index 0000000000000..ffdbf2ab062e6 --- /dev/null +++ b/contrib/pg_tde/t/013_crash_recovery.pl @@ -0,0 +1,107 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use File::Basename; +use Test::More; +use lib 't'; +use pgtde; + +# ensure we start with a clean key provider file +unlink('/tmp/crash_recovery.per'); + +PGTDE::setup_files_dir(basename($0)); + +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf( + 'postgresql.conf', q{ +checkpoint_timeout = 1h +shared_preload_libraries = 'pg_tde' +}); +$node->start; + +PGTDE::psql($node, 'postgres', 'CREATE EXTENSION IF NOT EXISTS pg_tde;'); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_add_global_key_provider_file('global_keyring', '/tmp/crash_recovery.per');" +); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_server_key_using_global_key_provider('wal_encryption_key', 'global_keyring');" +); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_add_database_key_provider_file('db_keyring', '/tmp/crash_recovery.per');" +); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_key_using_database_key_provider('db_key', 'db_keyring');" +); + +PGTDE::psql($node, 'postgres', + "CREATE TABLE test_enc (x int PRIMARY KEY) USING tde_heap;"); +PGTDE::psql($node, 'postgres', "INSERT INTO test_enc (x) VALUES (1), (2);"); + +PGTDE::psql($node, 'postgres', + "CREATE TABLE test_plain (x int PRIMARY KEY) USING heap;"); +PGTDE::psql($node, 'postgres', "INSERT INTO test_plain (x) VALUES (3), (4);"); + +PGTDE::psql($node, 'postgres', "ALTER SYSTEM SET pg_tde.wal_encrypt = 'on';"); + +PGTDE::append_to_result_file("-- kill -9"); +$node->kill9; + +PGTDE::append_to_result_file("-- server start"); +$node->start; + +PGTDE::append_to_result_file("-- rotate wal key"); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_server_key_using_global_key_provider('wal_encryption_key_1', 'global_keyring');" +); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_key_using_database_key_provider('db_key_1', 'db_keyring');" +); +PGTDE::psql($node, 'postgres', "INSERT INTO test_enc (x) VALUES (3), (4);"); +PGTDE::append_to_result_file("-- kill -9"); +$node->kill9; +PGTDE::append_to_result_file("-- server start"); +PGTDE::append_to_result_file( + "-- check that pg_tde_save_principal_key_redo hasn't destroyed a WAL key created during the server start" +); +$node->start; + +PGTDE::append_to_result_file("-- rotate wal key"); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_server_key_using_global_key_provider('wal_encryption_key_2', 'global_keyring');" +); +PGTDE::psql($node, 'postgres', + "SELECT pg_tde_set_key_using_database_key_provider('db_key_2', 'db_keyring');" +); +PGTDE::psql($node, 'postgres', "INSERT INTO test_enc (x) VALUES (5), (6);"); +PGTDE::append_to_result_file("-- kill -9"); +$node->kill9; +PGTDE::append_to_result_file("-- server start"); +PGTDE::append_to_result_file( + "-- check that the key rotation hasn't destroyed a WAL key created during the server start" +); +$node->start; + +PGTDE::psql($node, 'postgres', "TABLE test_enc;"); + +PGTDE::psql($node, 'postgres', + "CREATE TABLE test_enc2 (x int PRIMARY KEY) USING tde_heap;"); +PGTDE::append_to_result_file("-- kill -9"); +$node->kill9; +PGTDE::append_to_result_file("-- server start"); +PGTDE::append_to_result_file( + "-- check redo of the smgr internal key creation when the key is on disk" +); +$node->start; + +$node->stop; + +# Compare the expected and out file +my $compare = PGTDE->compare_results(); + +is($compare, 0, + "Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files." +); + +done_testing(); diff --git a/src/bin/pg_waldump/t/003_basic_encrypted.pl b/contrib/pg_tde/t/014_pg_waldump_basic.pl similarity index 83% rename from src/bin/pg_waldump/t/003_basic_encrypted.pl rename to contrib/pg_tde/t/014_pg_waldump_basic.pl index 1fd4fad129ad9..09315cf39d421 100644 --- a/src/bin/pg_waldump/t/003_basic_encrypted.pl +++ b/contrib/pg_tde/t/014_pg_waldump_basic.pl @@ -27,8 +27,12 @@ $node->start; $node->safe_psql('postgres', "CREATE EXTENSION IF NOT EXISTS pg_tde;"); -$node->safe_psql('postgres', "SELECT pg_tde_add_global_key_provider_file('file-keyring-wal','/tmp/pg_tde_test_keyring-wal.per');");; -$node->safe_psql('postgres', "SELECT pg_tde_set_server_principal_key('global-db-principal-key', 'file-keyring-wal');"); +$node->safe_psql('postgres', + "SELECT pg_tde_add_global_key_provider_file('file-keyring-wal','/tmp/pg_tde_test_keyring-wal.per');" +); +$node->safe_psql('postgres', + "SELECT pg_tde_set_server_key_using_global_key_provider('server-key', 'file-keyring-wal');" +); $node->append_conf( 'postgresql.conf', q{ @@ -128,48 +132,74 @@ [ 'pg_waldump', 'foo', 'bar' ], qr/error: could not locate WAL file "foo"/, 'start file not found'); -command_like([ 'pg_waldump', '-k', $node->data_dir. '/pg_tde', $node->data_dir . '/pg_wal/' . $start_walfile ], - qr/./, 'runs with start segment specified'); +command_like( + [ + 'pg_waldump', '-k', + $node->data_dir . '/pg_tde', + $node->data_dir . '/pg_wal/' . $start_walfile + ], + qr/./, + 'runs with start segment specified'); command_fails_like( - [ 'pg_waldump', '-k', $node->data_dir. '/pg_tde', $node->data_dir . '/pg_wal/' . $start_walfile, 'bar' ], + [ + 'pg_waldump', '-k', + $node->data_dir . '/pg_tde', + $node->data_dir . '/pg_wal/' . $start_walfile, 'bar' + ], qr/error: could not open file "bar"/, 'end file not found'); command_like( [ 'pg_waldump', - '-k', $node->data_dir. '/pg_tde', + '-k', + $node->data_dir . '/pg_tde', $node->data_dir . '/pg_wal/' . $start_walfile, $node->data_dir . '/pg_wal/' . $end_walfile ], qr/./, 'runs with start and end segment specified'); command_fails_like( - [ 'pg_waldump', '-p', $node->data_dir, '-k', $node->data_dir. '/pg_tde' ], + [ + 'pg_waldump', '-p', $node->data_dir, '-k', + $node->data_dir . '/pg_tde' + ], qr/error: no start WAL location given/, 'path option requires start location'); command_like( [ - 'pg_waldump', '-p', $node->data_dir, '--start', - $start_lsn, '--end', $end_lsn, - '-k', $node->data_dir. '/pg_tde' + 'pg_waldump', '-p', + $node->data_dir, '--start', + $start_lsn, '--end', + $end_lsn, '-k', + $node->data_dir . '/pg_tde' ], qr/./, 'runs with path option and start and end locations'); command_fails_like( - [ 'pg_waldump', '-k', $node->data_dir. '/pg_tde', '-p', $node->data_dir, '--start', $start_lsn ], + [ + 'pg_waldump', '-k', + $node->data_dir . '/pg_tde', '-p', + $node->data_dir, '--start', + $start_lsn + ], qr/error: error in WAL record at/, 'falling off the end of the WAL results in an error'); command_like( [ - 'pg_waldump', '--quiet', - '-k', $node->data_dir. '/pg_tde', + 'pg_waldump', '--quiet', '-k', + $node->data_dir . '/pg_tde', $node->data_dir . '/pg_wal/' . $start_walfile ], qr/^$/, 'no output with --quiet option'); command_fails_like( - [ 'pg_waldump', '--quiet', '-k', $node->data_dir. '/pg_tde', '-p', $node->data_dir, '--start', $start_lsn ], + [ + 'pg_waldump', '--quiet', + '-k', $node->data_dir . '/pg_tde', + '-p', $node->data_dir, + '--start', $start_lsn + ], qr/error: error in WAL record at/, 'errors are shown with --quiet'); @@ -187,9 +217,8 @@ my (@cmd, $stdout, $stderr, $result); @cmd = ( - 'pg_waldump', '-k', $node->data_dir. '/pg_tde', - '--start', $new_start, - $node->data_dir . '/pg_wal/' . $start_walfile); + 'pg_waldump', '-k', $node->data_dir . '/pg_tde', + '--start', $new_start, $node->data_dir . '/pg_wal/' . $start_walfile); $result = IPC::Run::run \@cmd, '>', \$stdout, '2>', \$stderr; ok($result, "runs with start segment and start LSN specified"); like($stderr, qr/first record is after/, 'info message printed'); @@ -206,8 +235,11 @@ sub test_pg_waldump my (@cmd, $stdout, $stderr, $result, @lines); @cmd = ( - 'pg_waldump', '-k', $node->data_dir. '/pg_tde', '-p', $node->data_dir, - '--start', $start_lsn, '--end', $end_lsn); + 'pg_waldump', '-k', + $node->data_dir . '/pg_tde', '-p', + $node->data_dir, '--start', + $start_lsn, '--end', + $end_lsn); push @cmd, @opts; $result = IPC::Run::run \@cmd, '>', \$stdout, '2>', \$stderr; ok($result, "pg_waldump @opts: runs ok"); diff --git a/src/bin/pg_waldump/t/004_save_fullpage_encrypted.pl b/contrib/pg_tde/t/015_pg_waldump_fullpage.pl similarity index 91% rename from src/bin/pg_waldump/t/004_save_fullpage_encrypted.pl rename to contrib/pg_tde/t/015_pg_waldump_fullpage.pl index 9b88ec89a9b1b..877ff2d34599b 100644 --- a/src/bin/pg_waldump/t/004_save_fullpage_encrypted.pl +++ b/contrib/pg_tde/t/015_pg_waldump_fullpage.pl @@ -41,8 +41,12 @@ sub get_block_lsn $node->start; $node->safe_psql('postgres', "CREATE EXTENSION IF NOT EXISTS pg_tde;"); -$node->safe_psql('postgres', "SELECT pg_tde_add_global_key_provider_file('file-keyring-wal','/tmp/pg_tde_test_keyring-wal.per');");; -$node->safe_psql('postgres', "SELECT pg_tde_set_server_principal_key('global-db-principal-key', 'file-keyring-wal');"); +$node->safe_psql('postgres', + "SELECT pg_tde_add_global_key_provider_file('file-keyring-wal','/tmp/pg_tde_test_keyring-wal.per');" +); +$node->safe_psql('postgres', + "SELECT pg_tde_set_server_key_using_global_key_provider('server-key', 'file-keyring-wal');" +); $node->append_conf( 'postgresql.conf', q{ @@ -84,7 +88,7 @@ sub get_block_lsn $node->command_ok( [ 'pg_waldump', '--quiet', - '-k', $node->data_dir. '/pg_tde', + '-k', $node->data_dir . '/pg_tde', '--save-fullpage', "$tmp_folder/raw", '--relation', $relation, $walfile diff --git a/contrib/pg_tde/t/expected/001_basic.out b/contrib/pg_tde/t/expected/001_basic.out index 0ea866e83ebc8..c1e385741b99f 100644 --- a/contrib/pg_tde/t/expected/001_basic.out +++ b/contrib/pg_tde/t/expected/001_basic.out @@ -1,16 +1,43 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; SELECT extname, extversion FROM pg_extension WHERE extname = 'pg_tde'; -pg_tde|1.0-rc + extname | extversion +---------+------------ + pg_tde | 1.0-rc +(1 row) + +CREATE TABLE test_enc(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap; +psql::1: ERROR: principal key not configured +HINT: create one using pg_tde_set_key before using encrypted tables -- server restart +SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 +(1 row) + +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- + +(1 row) + CREATE TABLE test_enc(id SERIAL,k VARCHAR(32),PRIMARY KEY (id)) USING tde_heap; INSERT INTO test_enc (k) VALUES ('foobar'),('barfoo'); SELECT * FROM test_enc ORDER BY id ASC; -1|foobar -2|barfoo + id | k +----+-------- + 1 | foobar + 2 | barfoo +(2 rows) + -- server restart SELECT * FROM test_enc ORDER BY id ASC; -1|foobar -2|barfoo + id | k +----+-------- + 1 | foobar + 2 | barfoo +(2 rows) + TABLEFILE FOUND: yes CONTAINS FOO (should be empty): diff --git a/contrib/pg_tde/t/expected/002_rotate_key.out b/contrib/pg_tde/t/expected/002_rotate_key.out index 86b9646088a09..12129902d2d3d 100644 --- a/contrib/pg_tde/t/expected/002_rotate_key.out +++ b/contrib/pg_tde/t/expected/002_rotate_key.out @@ -1,94 +1,203 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; +CREATE TABLE test_enc(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap; +psql::1: ERROR: principal key not configured +HINT: create one using pg_tde_set_key before using encrypted tables -- server restart -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); -1 -SELECT pg_tde_add_key_provider_file('file-2','/tmp/pg_tde_test_keyring_2.per'); -2 +SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 +(1 row) + +SELECT pg_tde_add_database_key_provider_file('file-2','/tmp/pg_tde_test_keyring_2.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 2 +(1 row) + SELECT pg_tde_add_global_key_provider_file('file-2','/tmp/pg_tde_test_keyring_2g.per'); --1 + pg_tde_add_global_key_provider_file +------------------------------------- + -1 +(1 row) + SELECT pg_tde_add_global_key_provider_file('file-3','/tmp/pg_tde_test_keyring_3.per'); --2 -SELECT pg_tde_list_all_key_providers(); -(1,file-vault,file,"{""type"" : ""file"", ""path"" : ""/tmp/pg_tde_test_keyring.per""}") -(2,file-2,file,"{""type"" : ""file"", ""path"" : ""/tmp/pg_tde_test_keyring_2.per""}") -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); + pg_tde_add_global_key_provider_file +------------------------------------- + -2 +(1 row) + +SELECT pg_tde_list_all_database_key_providers(); + pg_tde_list_all_database_key_providers +------------------------------------------------------------------------------------------ + (1,file-vault,file,"{""type"" : ""file"", ""path"" : ""/tmp/pg_tde_test_keyring.per""}") + (2,file-2,file,"{""type"" : ""file"", ""path"" : ""/tmp/pg_tde_test_keyring_2.per""}") +(2 rows) + +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- + +(1 row) CREATE TABLE test_enc(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap; INSERT INTO test_enc (k) VALUES (5),(6); SELECT * FROM test_enc ORDER BY id ASC; -1|5 -2|6 -0 + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + +SELECT pg_tde_set_key_using_database_key_provider('rotated-key1'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- + +(1 row) + SELECT * FROM test_enc ORDER BY id ASC; -1|5 -2|6 + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + -- server restart -SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_principal_key_info(); -1|file-vault|rotated-principal-key1 -SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_global_principal_key_info(); +SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_key_info(); + key_provider_id | key_provider_name | key_name +-----------------+-------------------+-------------- + 1 | file-vault | rotated-key1 +(1 row) + +SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_server_key_info(); psql::1: ERROR: Principal key does not exists for the database -HINT: Use set_principal_key interface to set the principal key +HINT: Use set_key interface to set the principal key SELECT * FROM test_enc ORDER BY id ASC; -1|5 -2|6 -SELECT pg_tde_set_principal_key('rotated-principal-key2','file-2'); + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + +SELECT pg_tde_set_key_using_database_key_provider('rotated-key2','file-2'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- + +(1 row) SELECT * FROM test_enc ORDER BY id ASC; -1|5 -2|6 + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + -- server restart -SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_principal_key_info(); -2|file-2|rotated-principal-key2 -SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_global_principal_key_info(); +SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_key_info(); + key_provider_id | key_provider_name | key_name +-----------------+-------------------+-------------- + 2 | file-2 | rotated-key2 +(1 row) + +SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_server_key_info(); psql::1: ERROR: Principal key does not exists for the database -HINT: Use set_principal_key interface to set the principal key +HINT: Use set_key interface to set the principal key SELECT * FROM test_enc ORDER BY id ASC; -1|5 -2|6 -SELECT pg_tde_set_global_principal_key('rotated-principal-key', 'file-3', false); + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + +SELECT pg_tde_set_key_using_global_key_provider('rotated-key', 'file-3', false); + pg_tde_set_key_using_global_key_provider +------------------------------------------ + +(1 row) SELECT * FROM test_enc ORDER BY id ASC; -1|5 -2|6 + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + -- server restart -SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_principal_key_info(); --2|file-3|rotated-principal-key -SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_global_principal_key_info(); +SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_key_info(); + key_provider_id | key_provider_name | key_name +-----------------+-------------------+------------- + -2 | file-3 | rotated-key +(1 row) + +SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_server_key_info(); psql::1: ERROR: Principal key does not exists for the database -HINT: Use set_principal_key interface to set the principal key +HINT: Use set_key interface to set the principal key SELECT * FROM test_enc ORDER BY id ASC; -1|5 -2|6 -SELECT pg_tde_set_global_principal_key('rotated-principal-keyX', 'file-2', false); + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + +SELECT pg_tde_set_key_using_global_key_provider('rotated-keyX', 'file-2', false); + pg_tde_set_key_using_global_key_provider +------------------------------------------ + +(1 row) SELECT * FROM test_enc ORDER BY id ASC; -1|5 -2|6 + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + -- server restart -SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_principal_key_info(); --1|file-2|rotated-principal-keyX -SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_global_principal_key_info(); +SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_key_info(); + key_provider_id | key_provider_name | key_name +-----------------+-------------------+-------------- + -1 | file-2 | rotated-keyX +(1 row) + +SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_server_key_info(); psql::1: ERROR: Principal key does not exists for the database -HINT: Use set_principal_key interface to set the principal key +HINT: Use set_key interface to set the principal key SELECT * FROM test_enc ORDER BY id ASC; -1|5 -2|6 + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + ALTER SYSTEM SET pg_tde.inherit_global_providers = OFF; -- server restart +SELECT pg_tde_set_key_using_global_key_provider('rotated-keyX2', 'file-2', false); psql::1: ERROR: Usage of global key providers is disabled. Enable it with pg_tde.inherit_global_providers = ON -SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_principal_key_info(); --1|file-2|rotated-principal-keyX -SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_global_principal_key_info(); +SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_key_info(); + key_provider_id | key_provider_name | key_name +-----------------+-------------------+-------------- + -1 | file-2 | rotated-keyX +(1 row) + +SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_server_key_info(); psql::1: ERROR: Principal key does not exists for the database -HINT: Use set_principal_key interface to set the principal key -SELECT pg_tde_set_principal_key('rotated-principal-key2','file-2'); +HINT: Use set_key interface to set the principal key +SELECT pg_tde_set_key_using_database_key_provider('rotated-key2','file-2'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- + +(1 row) -SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_principal_key_info(); -2|file-2|rotated-principal-key2 -SELECT key_provider_id, key_provider_name, principal_key_name FROM pg_tde_global_principal_key_info(); +SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_key_info(); + key_provider_id | key_provider_name | key_name +-----------------+-------------------+-------------- + 2 | file-2 | rotated-key2 +(1 row) + +SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_server_key_info(); psql::1: ERROR: Principal key does not exists for the database -HINT: Use set_principal_key interface to set the principal key +HINT: Use set_key interface to set the principal key DROP TABLE test_enc; ALTER SYSTEM RESET pg_tde.inherit_global_providers; +-- server restart DROP EXTENSION pg_tde CASCADE; - diff --git a/contrib/pg_tde/t/expected/003_remote_config.out b/contrib/pg_tde/t/expected/003_remote_config.out index 35bcd3f49651a..fc4208ba360ed 100644 --- a/contrib/pg_tde/t/expected/003_remote_config.out +++ b/contrib/pg_tde/t/expected/003_remote_config.out @@ -1,12 +1,32 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; +SELECT pg_tde_add_database_key_provider_file('file-provider', json_object( 'type' VALUE 'remote', 'url' VALUE 'http://localhost:8888/hello' )); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 +(1 row) + +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-provider'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- + +(1 row) + CREATE TABLE test_enc2(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap; INSERT INTO test_enc2 (k) VALUES (5),(6); SELECT * FROM test_enc2 ORDER BY id ASC; -1|5 -2|6 + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + -- server restart SELECT * FROM test_enc2 ORDER BY id ASC; -1|5 -2|6 + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + DROP TABLE test_enc2; DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/t/expected/004_file_config.out b/contrib/pg_tde/t/expected/004_file_config.out index e5f705d3137c3..878e2d3621ea0 100644 --- a/contrib/pg_tde/t/expected/004_file_config.out +++ b/contrib/pg_tde/t/expected/004_file_config.out @@ -1,12 +1,32 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; +SELECT pg_tde_add_database_key_provider_file('file-provider', json_object( 'type' VALUE 'file', 'path' VALUE '/tmp/datafile-location' )); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 +(1 row) + +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-provider'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- + +(1 row) + CREATE TABLE test_enc1(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap; INSERT INTO test_enc1 (k) VALUES (5),(6); SELECT * FROM test_enc1 ORDER BY id ASC; -1|5 -2|6 + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + -- server restart SELECT * FROM test_enc1 ORDER BY id ASC; -1|5 -2|6 + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + DROP TABLE test_enc1; DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/t/expected/006_remote_vault_config.out b/contrib/pg_tde/t/expected/006_remote_vault_config.out index 35bcd3f49651a..8fe7b12360bc0 100644 --- a/contrib/pg_tde/t/expected/006_remote_vault_config.out +++ b/contrib/pg_tde/t/expected/006_remote_vault_config.out @@ -1,12 +1,32 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; +SELECT pg_tde_add_database_key_provider_vault_v2('vault-provider', json_object( 'type' VALUE 'remote', 'url' VALUE 'http://localhost:8889/token' ), json_object( 'type' VALUE 'remote', 'url' VALUE 'http://localhost:8889/url' ), to_json('secret'::text), NULL); + pg_tde_add_database_key_provider_vault_v2 +------------------------------------------- + 1 +(1 row) + +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','vault-provider'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- + +(1 row) + CREATE TABLE test_enc2(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap; INSERT INTO test_enc2 (k) VALUES (5),(6); SELECT * FROM test_enc2 ORDER BY id ASC; -1|5 -2|6 + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + -- server restart SELECT * FROM test_enc2 ORDER BY id ASC; -1|5 -2|6 + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + DROP TABLE test_enc2; DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/t/expected/007_tde_heap.out b/contrib/pg_tde/t/expected/007_tde_heap.out index 867f714c9fb16..6a5aef32bf731 100644 --- a/contrib/pg_tde/t/expected/007_tde_heap.out +++ b/contrib/pg_tde/t/expected/007_tde_heap.out @@ -1,62 +1,119 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; +CREATE TABLE test_enc(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING tde_heap; +psql::1: ERROR: principal key not configured +HINT: create one using pg_tde_set_key before using encrypted tables -- server restart +SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 +(1 row) + +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- + +(1 row) + CREATE TABLE test_enc1(id SERIAL,k VARCHAR(32),PRIMARY KEY (id)) USING tde_heap; INSERT INTO test_enc1 (k) VALUES ('foobar'),('barfoo'); SELECT * FROM test_enc1 ORDER BY id ASC; -1|foobar -2|barfoo + id | k +----+-------- + 1 | foobar + 2 | barfoo +(2 rows) + CREATE TABLE test_enc2(id SERIAL,k VARCHAR(32),PRIMARY KEY (id)); INSERT INTO test_enc2 (k) VALUES ('foobar'),('barfoo'); ALTER TABLE test_enc2 SET ACCESS METHOD tde_heap; SELECT * FROM test_enc2 ORDER BY id ASC; -1|foobar -2|barfoo + id | k +----+-------- + 1 | foobar + 2 | barfoo +(2 rows) + SET default_table_access_method = "tde_heap"; CREATE TABLE test_enc3(id SERIAL,k VARCHAR(32),PRIMARY KEY (id)); INSERT INTO test_enc3 (k) VALUES ('foobar'),('barfoo'); SELECT * FROM test_enc3 ORDER BY id ASC; -1|foobar -2|barfoo + id | k +----+-------- + 1 | foobar + 2 | barfoo +(2 rows) + +CREATE TABLE test_enc4(id SERIAL,k VARCHAR(32),PRIMARY KEY (id)) USING heap; INSERT INTO test_enc4 (k) VALUES ('foobar'),('barfoo'); +SET default_table_access_method = "tde_heap"; ALTER TABLE test_enc4 SET ACCESS METHOD DEFAULT; SELECT * FROM test_enc4 ORDER BY id ASC; -1|foobar -2|barfoo + id | k +----+-------- + 1 | foobar + 2 | barfoo +(2 rows) + CREATE TABLE test_enc5(id SERIAL,k VARCHAR(32),PRIMARY KEY (id)) USING tde_heap; INSERT INTO test_enc5 (k) VALUES ('foobar'),('barfoo'); CHECKPOINT; TRUNCATE test_enc5; INSERT INTO test_enc5 (k) VALUES ('foobar'),('barfoo'); SELECT * FROM test_enc5 ORDER BY id ASC; -3|foobar -4|barfoo + id | k +----+-------- + 3 | foobar + 4 | barfoo +(2 rows) + -- server restart ########################### SELECT * FROM test_enc1 ORDER BY id ASC; -1|foobar -2|barfoo + id | k +----+-------- + 1 | foobar + 2 | barfoo +(2 rows) + TABLEFILE FOR test_enc1 FOUND: yes CONTAINS FOO (should be empty): ########################### SELECT * FROM test_enc2 ORDER BY id ASC; -1|foobar -2|barfoo + id | k +----+-------- + 1 | foobar + 2 | barfoo +(2 rows) + TABLEFILE FOR test_enc2 FOUND: yes CONTAINS FOO (should be empty): ########################### SELECT * FROM test_enc3 ORDER BY id ASC; -1|foobar -2|barfoo + id | k +----+-------- + 1 | foobar + 2 | barfoo +(2 rows) + TABLEFILE FOR test_enc3 FOUND: yes CONTAINS FOO (should be empty): ########################### SELECT * FROM test_enc4 ORDER BY id ASC; -1|foobar -2|barfoo + id | k +----+-------- + 1 | foobar + 2 | barfoo +(2 rows) + TABLEFILE FOR test_enc4 FOUND: yes CONTAINS FOO (should be empty): ########################### SELECT * FROM test_enc5 ORDER BY id ASC; -3|foobar -4|barfoo + id | k +----+-------- + 3 | foobar + 4 | barfoo +(2 rows) + TABLEFILE FOR test_enc5 FOUND: yes CONTAINS FOO (should be empty): TABLEFILE2 FOUND: yes diff --git a/contrib/pg_tde/t/expected/008_key_rotate_tablespace.out b/contrib/pg_tde/t/expected/008_key_rotate_tablespace.out index 5ed45191455e8..7d88a74e1c47d 100644 --- a/contrib/pg_tde/t/expected/008_key_rotate_tablespace.out +++ b/contrib/pg_tde/t/expected/008_key_rotate_tablespace.out @@ -1,25 +1,17 @@ +SET allow_in_place_tablespaces = true; CREATE TABLESPACE test_tblspace LOCATION ''; +CREATE DATABASE tbc TABLESPACE = test_tblspace; CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); -1 -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); +SELECT pg_tde_add_database_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 +(1 row) -CREATE TABLE country_table ( - country_id serial primary key, - country_name text unique not null, - continent text not null -) USING tde_heap; -INSERT INTO country_table (country_name, continent) - VALUES ('Japan', 'Asia'), - ('UK', 'Europe'), - ('USA', 'North America'); -SELECT * FROM country_table; -1|Japan|Asia -2|UK|Europe -3|USA|North America -CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); -1 -SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); +SELECT pg_tde_set_key_using_database_key_provider('test-db-key','file-vault'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- + +(1 row) CREATE TABLE country_table ( country_id serial primary key, @@ -31,14 +23,29 @@ INSERT INTO country_table (country_name, continent) ('UK', 'Europe'), ('USA', 'North America'); SELECT * FROM country_table; -1|Japan|Asia -2|UK|Europe -3|USA|North America + country_id | country_name | continent +------------+--------------+--------------- + 1 | Japan | Asia + 2 | UK | Europe + 3 | USA | North America +(3 rows) + +SELECT pg_tde_set_key_using_database_key_provider('new-k', 'file-vault'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- + +(1 row) + -- server restart SELECT * FROM country_table; -1|Japan|Asia -2|UK|Europe -3|USA|North America + country_id | country_name | continent +------------+--------------+--------------- + 1 | Japan | Asia + 2 | UK | Europe + 3 | USA | North America +(3 rows) + DROP EXTENSION pg_tde CASCADE; +psql::1: NOTICE: drop cascades to table country_table DROP DATABASE tbc; DROP TABLESPACE test_tblspace; diff --git a/contrib/pg_tde/t/expected/009_wal_encrypt.out b/contrib/pg_tde/t/expected/009_wal_encrypt.out index 67b4e5e56d94e..c0df1e2a03110 100644 --- a/contrib/pg_tde/t/expected/009_wal_encrypt.out +++ b/contrib/pg_tde/t/expected/009_wal_encrypt.out @@ -1,11 +1,85 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; SELECT pg_tde_add_global_key_provider_file('file-keyring-010','/tmp/pg_tde_test_keyring010.per'); --1 -SELECT pg_tde_set_server_principal_key('global-db-principal-key', 'file-keyring-010'); + pg_tde_add_global_key_provider_file +------------------------------------- + -1 +(1 row) +SELECT pg_tde_set_server_key_using_global_key_provider('server-key', 'file-keyring-010'); + pg_tde_set_server_key_using_global_key_provider +------------------------------------------------- + +(1 row) + +ALTER SYSTEM SET pg_tde.wal_encrypt = on; +-- server restart with wal encryption +SHOW pg_tde.wal_encrypt; + pg_tde.wal_encrypt +-------------------- + on +(1 row) + +SELECT slot_name FROM pg_create_logical_replication_slot('tde_slot', 'test_decoding'); + slot_name +----------- + tde_slot +(1 row) + +CREATE TABLE test_wal (id SERIAL, k INTEGER, PRIMARY KEY (id)); +INSERT INTO test_wal (k) VALUES (1), (2); +ALTER SYSTEM SET pg_tde.wal_encrypt = off; +-- server restart without wal encryption +SHOW pg_tde.wal_encrypt; + pg_tde.wal_encrypt +-------------------- + off +(1 row) + +INSERT INTO test_wal (k) VALUES (3), (4); +ALTER SYSTEM SET pg_tde.wal_encrypt = on; -- server restart with wal encryption SHOW pg_tde.wal_encrypt; -on -CREATE TABLE test_wal(id SERIAL,k INTEGER,PRIMARY KEY (id)); -CHECKPOINT; + pg_tde.wal_encrypt +-------------------- + on +(1 row) + +INSERT INTO test_wal (k) VALUES (5), (6); +-- server restart with still wal encryption +SHOW pg_tde.wal_encrypt; + pg_tde.wal_encrypt +-------------------- + on +(1 row) + +INSERT INTO test_wal (k) VALUES (7), (8); +SELECT data FROM pg_logical_slot_get_changes('tde_slot', NULL, NULL); + data +----------------------------------------------------------- + BEGIN 739 + COMMIT 739 + BEGIN 740 + table public.test_wal: INSERT: id[integer]:1 k[integer]:1 + table public.test_wal: INSERT: id[integer]:2 k[integer]:2 + COMMIT 740 + BEGIN 741 + table public.test_wal: INSERT: id[integer]:3 k[integer]:3 + table public.test_wal: INSERT: id[integer]:4 k[integer]:4 + COMMIT 741 + BEGIN 742 + table public.test_wal: INSERT: id[integer]:5 k[integer]:5 + table public.test_wal: INSERT: id[integer]:6 k[integer]:6 + COMMIT 742 + BEGIN 743 + table public.test_wal: INSERT: id[integer]:7 k[integer]:7 + table public.test_wal: INSERT: id[integer]:8 k[integer]:8 + COMMIT 743 +(18 rows) + +SELECT pg_drop_replication_slot('tde_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/t/expected/010_change_key_provider.out b/contrib/pg_tde/t/expected/010_change_key_provider.out index 507b8799177da..ca988c6259919 100644 --- a/contrib/pg_tde/t/expected/010_change_key_provider.out +++ b/contrib/pg_tde/t/expected/010_change_key_provider.out @@ -1,52 +1,124 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('file-vault', '/tmp/change_key_provider_1.per'); -1 -SELECT pg_tde_list_all_key_providers(); -(1,file-vault,file,"{""type"" : ""file"", ""path"" : ""/tmp/change_key_provider_1.per""}") -SELECT pg_tde_set_principal_key('test-key', 'file-vault'); +SELECT pg_tde_add_database_key_provider_file('file-vault', '/tmp/change_key_provider_1.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 +(1 row) + +SELECT pg_tde_list_all_database_key_providers(); + pg_tde_list_all_database_key_providers +-------------------------------------------------------------------------------------------- + (1,file-vault,file,"{""type"" : ""file"", ""path"" : ""/tmp/change_key_provider_1.per""}") +(1 row) + +SELECT pg_tde_set_key_using_database_key_provider('test-key', 'file-vault'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- + +(1 row) CREATE TABLE test_enc (id serial, k integer, PRIMARY KEY (id)) USING tde_heap; INSERT INTO test_enc (k) VALUES (5), (6); -SELECT pg_tde_verify_principal_key(); +SELECT pg_tde_verify_key(); + pg_tde_verify_key +------------------- + +(1 row) SELECT pg_tde_is_encrypted('test_enc'); -t + pg_tde_is_encrypted +--------------------- + t +(1 row) + SELECT * FROM test_enc ORDER BY id; -1|5 -2|6 + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + -- mv /tmp/change_key_provider_1.per /tmp/change_key_provider_2.per -SELECT pg_tde_change_key_provider_file('file-vault', '/tmp/change_key_provider_2.per'); -1 -SELECT pg_tde_list_all_key_providers(); -(1,file-vault,file,"{""type"" : ""file"", ""path"" : ""/tmp/change_key_provider_2.per""}") -SELECT pg_tde_verify_principal_key(); +SELECT pg_tde_change_database_key_provider_file('file-vault', '/tmp/change_key_provider_2.per'); + pg_tde_change_database_key_provider_file +------------------------------------------ + 1 +(1 row) + +SELECT pg_tde_list_all_database_key_providers(); + pg_tde_list_all_database_key_providers +-------------------------------------------------------------------------------------------- + (1,file-vault,file,"{""type"" : ""file"", ""path"" : ""/tmp/change_key_provider_2.per""}") +(1 row) + +SELECT pg_tde_verify_key(); + pg_tde_verify_key +------------------- + +(1 row) SELECT pg_tde_is_encrypted('test_enc'); -t + pg_tde_is_encrypted +--------------------- + t +(1 row) + SELECT * FROM test_enc ORDER BY id; -1|5 -2|6 + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + -- server restart -SELECT pg_tde_verify_principal_key(); +SELECT pg_tde_verify_key(); + pg_tde_verify_key +------------------- + +(1 row) SELECT pg_tde_is_encrypted('test_enc'); -t + pg_tde_is_encrypted +--------------------- + t +(1 row) + SELECT * FROM test_enc ORDER BY id; -1|5 -2|6 -SELECT pg_tde_change_key_provider_file('file-vault', '/tmp/change_key_provider_3.per'); -1 -SELECT pg_tde_list_all_key_providers(); -(1,file-vault,file,"{""type"" : ""file"", ""path"" : ""/tmp/change_key_provider_3.per""}") -SELECT pg_tde_verify_principal_key(); + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + +SELECT pg_tde_change_database_key_provider_file('file-vault', '/tmp/change_key_provider_3.per'); + pg_tde_change_database_key_provider_file +------------------------------------------ + 1 +(1 row) + +SELECT pg_tde_list_all_database_key_providers(); + pg_tde_list_all_database_key_providers +-------------------------------------------------------------------------------------------- + (1,file-vault,file,"{""type"" : ""file"", ""path"" : ""/tmp/change_key_provider_3.per""}") +(1 row) + +SELECT pg_tde_verify_key(); psql::1: ERROR: failed to retrieve principal key test-key from keyring with ID 1 SELECT pg_tde_is_encrypted('test_enc'); -t + pg_tde_is_encrypted +--------------------- + t +(1 row) + SELECT * FROM test_enc ORDER BY id; -1|5 -2|6 + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + -- server restart -SELECT pg_tde_verify_principal_key(); +SELECT pg_tde_verify_key(); psql::1: ERROR: failed to retrieve principal key test-key from keyring with ID 1 SELECT pg_tde_is_encrypted('test_enc'); psql::1: ERROR: failed to retrieve principal key test-key from keyring with ID 1 @@ -54,47 +126,100 @@ SELECT * FROM test_enc ORDER BY id; psql::1: ERROR: failed to retrieve principal key test-key from keyring with ID 1 -- mv /tmp/change_key_provider_2.per /tmp/change_key_provider_3.per -- server restart -SELECT pg_tde_verify_principal_key(); +SELECT pg_tde_verify_key(); + pg_tde_verify_key +------------------- + +(1 row) SELECT pg_tde_is_encrypted('test_enc'); -t + pg_tde_is_encrypted +--------------------- + t +(1 row) + SELECT * FROM test_enc ORDER BY id; -1|5 -2|6 + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + DROP EXTENSION pg_tde CASCADE; psql::1: NOTICE: drop cascades to table test_enc CREATE EXTENSION IF NOT EXISTS pg_tde; -SELECT pg_tde_add_key_provider_file('file-vault', '/tmp/change_key_provider_4.per'); -1 -0 +SELECT pg_tde_add_database_key_provider_file('file-vault', '/tmp/change_key_provider_4.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 +(1 row) + +SELECT pg_tde_set_key_using_database_key_provider('test-key', 'file-vault'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- + +(1 row) + CREATE TABLE test_enc (id serial, k integer, PRIMARY KEY (id)) USING tde_heap; INSERT INTO test_enc (k) VALUES (5), (6); -SELECT pg_tde_verify_principal_key(); +SELECT pg_tde_verify_key(); + pg_tde_verify_key +------------------- + +(1 row) SELECT pg_tde_is_encrypted('test_enc'); -t + pg_tde_is_encrypted +--------------------- + t +(1 row) + SELECT * FROM test_enc ORDER BY id; -1|5 -2|6 -SELECT pg_tde_change_key_provider_file('file-vault', '/tmp/change_key_provider_3.per'); -1 --- server restart -SELECT pg_tde_verify_principal_key(); + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) +SELECT pg_tde_change_database_key_provider_file('file-vault', '/tmp/change_key_provider_3.per'); + pg_tde_change_database_key_provider_file +------------------------------------------ + 1 +(1 row) +-- server restart +SELECT pg_tde_verify_key(); +psql::1: ERROR: Failed to verify principal key header for key test-key, incorrect principal key or corrupted key file SELECT pg_tde_is_encrypted('test_enc'); -psql::1: ERROR: Failed to decrypt key, incorrect principal key or corrupted key file +psql::1: ERROR: Failed to verify principal key header for key test-key, incorrect principal key or corrupted key file SELECT * FROM test_enc ORDER BY id; -psql::1: ERROR: Failed to decrypt key, incorrect principal key or corrupted key file -SELECT pg_tde_change_key_provider_file('file-vault', '/tmp/change_key_provider_4.per'); -1 --- server restart -SELECT pg_tde_verify_principal_key(); +psql::1: ERROR: Failed to verify principal key header for key test-key, incorrect principal key or corrupted key file +CREATE TABLE test_enc2 (id serial, k integer, PRIMARY KEY (id)) USING tde_heap; +psql::1: ERROR: Failed to verify principal key header for key test-key, incorrect principal key or corrupted key file +SELECT pg_tde_change_database_key_provider_file('file-vault', '/tmp/change_key_provider_4.per'); + pg_tde_change_database_key_provider_file +------------------------------------------ + 1 +(1 row) + +SELECT pg_tde_verify_key(); + pg_tde_verify_key +------------------- + +(1 row) SELECT pg_tde_is_encrypted('test_enc'); -t + pg_tde_is_encrypted +--------------------- + t +(1 row) + SELECT * FROM test_enc ORDER BY id; -1|5 -2|6 + id | k +----+--- + 1 | 5 + 2 | 6 +(2 rows) + DROP EXTENSION pg_tde CASCADE; psql::1: NOTICE: drop cascades to table test_enc diff --git a/contrib/pg_tde/t/expected/011_unlogged_tables.out b/contrib/pg_tde/t/expected/011_unlogged_tables.out new file mode 100644 index 0000000000000..71c52786b134f --- /dev/null +++ b/contrib/pg_tde/t/expected/011_unlogged_tables.out @@ -0,0 +1,24 @@ +CREATE EXTENSION IF NOT EXISTS pg_tde; +SELECT pg_tde_add_database_key_provider_file('file-vault', '/tmp/unlogged_tables.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 +(1 row) + +SELECT pg_tde_set_key_using_database_key_provider('test-key', 'file-vault'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- + +(1 row) + +CREATE UNLOGGED TABLE t (x int PRIMARY KEY) USING tde_heap; +INSERT INTO t SELECT generate_series(1, 4); +CHECKPOINT; +-- kill -9 +-- server start +TABLE t; + x +--- +(0 rows) + +INSERT INTO t SELECT generate_series(1, 4); diff --git a/contrib/pg_tde/t/expected/012_replication.out b/contrib/pg_tde/t/expected/012_replication.out new file mode 100644 index 0000000000000..0ea5f8470599b --- /dev/null +++ b/contrib/pg_tde/t/expected/012_replication.out @@ -0,0 +1,57 @@ +-- At primary +CREATE EXTENSION IF NOT EXISTS pg_tde; +SELECT pg_tde_add_database_key_provider_file('file-vault', '/tmp/unlogged_tables.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 +(1 row) + +SELECT pg_tde_set_key_using_database_key_provider('test-key', 'file-vault'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- + +(1 row) + +CREATE TABLE test_enc (x int PRIMARY KEY) USING tde_heap; +INSERT INTO test_enc (x) VALUES (1), (2); +CREATE TABLE test_plain (x int PRIMARY KEY) USING heap; +INSERT INTO test_plain (x) VALUES (3), (4); +-- At replica +SELECT pg_tde_is_encrypted('test_enc'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +SELECT pg_tde_is_encrypted('test_enc_pkey'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +SELECT * FROM test_enc ORDER BY x; + x +--- + 1 + 2 +(2 rows) + +SELECT pg_tde_is_encrypted('test_plain'); + pg_tde_is_encrypted +--------------------- + f +(1 row) + +SELECT pg_tde_is_encrypted('test_plain_pkey'); + pg_tde_is_encrypted +--------------------- + f +(1 row) + +SELECT * FROM test_plain ORDER BY x; + x +--- + 3 + 4 +(2 rows) + diff --git a/contrib/pg_tde/t/expected/013_crash_recovery.out b/contrib/pg_tde/t/expected/013_crash_recovery.out new file mode 100644 index 0000000000000..75cb4fa3bed81 --- /dev/null +++ b/contrib/pg_tde/t/expected/013_crash_recovery.out @@ -0,0 +1,81 @@ +CREATE EXTENSION IF NOT EXISTS pg_tde; +SELECT pg_tde_add_global_key_provider_file('global_keyring', '/tmp/crash_recovery.per'); + pg_tde_add_global_key_provider_file +------------------------------------- + -1 +(1 row) + +SELECT pg_tde_set_server_key_using_global_key_provider('wal_encryption_key', 'global_keyring'); + pg_tde_set_server_key_using_global_key_provider +------------------------------------------------- + +(1 row) + +SELECT pg_tde_add_database_key_provider_file('db_keyring', '/tmp/crash_recovery.per'); + pg_tde_add_database_key_provider_file +--------------------------------------- + 1 +(1 row) + +SELECT pg_tde_set_key_using_database_key_provider('db_key', 'db_keyring'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- + +(1 row) + +CREATE TABLE test_enc (x int PRIMARY KEY) USING tde_heap; +INSERT INTO test_enc (x) VALUES (1), (2); +CREATE TABLE test_plain (x int PRIMARY KEY) USING heap; +INSERT INTO test_plain (x) VALUES (3), (4); +ALTER SYSTEM SET pg_tde.wal_encrypt = 'on'; +-- kill -9 +-- server start +-- rotate wal key +SELECT pg_tde_set_server_key_using_global_key_provider('wal_encryption_key_1', 'global_keyring'); + pg_tde_set_server_key_using_global_key_provider +------------------------------------------------- + +(1 row) + +SELECT pg_tde_set_key_using_database_key_provider('db_key_1', 'db_keyring'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- + +(1 row) + +INSERT INTO test_enc (x) VALUES (3), (4); +-- kill -9 +-- server start +-- check that pg_tde_save_principal_key_redo hasn't destroyed a WAL key created during the server start +-- rotate wal key +SELECT pg_tde_set_server_key_using_global_key_provider('wal_encryption_key_2', 'global_keyring'); + pg_tde_set_server_key_using_global_key_provider +------------------------------------------------- + +(1 row) + +SELECT pg_tde_set_key_using_database_key_provider('db_key_2', 'db_keyring'); + pg_tde_set_key_using_database_key_provider +-------------------------------------------- + +(1 row) + +INSERT INTO test_enc (x) VALUES (5), (6); +-- kill -9 +-- server start +-- check that the key rotation hasn't destroyed a WAL key created during the server start +TABLE test_enc; + x +--- + 1 + 2 + 3 + 4 + 5 + 6 +(6 rows) + +CREATE TABLE test_enc2 (x int PRIMARY KEY) USING tde_heap; +-- kill -9 +-- server start +-- check redo of the smgr internal key creation when the key is on disk diff --git a/contrib/pg_tde/t/pgtde.pm b/contrib/pg_tde/t/pgtde.pm index 67b1709aad749..16b98c392a5b4 100644 --- a/contrib/pg_tde/t/pgtde.pm +++ b/contrib/pg_tde/t/pgtde.pm @@ -1,146 +1,84 @@ package PGTDE; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; + use File::Basename; use File::Compare; use Test::More; -our @ISA= qw( Exporter ); - -# These CAN be exported. -our @EXPORT = qw( pgtde_init_pg pgtde_start_pg pgtde_stop_pg pgtde_psql_cmd pgtde_setup_pg_tde pgtde_create_extension pgtde_drop_extension ); - -# Instance of pg server that would be spanwed by TAP testing. A new server will be created for each TAP test. -our $pg_node; - # Expected .out filename of TAP testcase being executed. These are already part of repo under t/expected/*. our $expected_filename_with_path; -# Major version of PG Server that we are using. -our $PG_MAJOR_VERSION; - # Result .out filename of TAP testcase being executed. Where needed, a new *.out will be created for each TAP test. our $out_filename_with_path; # Runtime output file that is used only for debugging purposes for comparison to PGSS, blocks and timings. our $debug_out_filename_with_path; -BEGIN { - # Get PG Server Major version from pg_config - $PG_MAJOR_VERSION = `pg_config --version | awk {'print \$2'} | cut -f1 -d"." | sed -e 's/[^0-9].*\$//g'`; - $PG_MAJOR_VERSION =~ s/^\s+|\s+$//g; - - # Depending upon PG server version load the required module at runtime when pgtde.pm is loaded. - my $node_module = $PG_MAJOR_VERSION > 14 ? "PostgreSQL::Test::Cluster" : "PostgresNode"; - my $node_module_file = $node_module; - $node_module_file =~ s[::][/]g; - $node_module_file .= '.pm'; - require $node_module_file; - $node_module->import; -} +my $expected_folder = "t/expected"; +my $results_folder = "t/results"; -sub pgtde_init_pg +sub psql { - print "Postgres major version: $PG_MAJOR_VERSION \n"; - - # For Server version 15 & above, spawn the server using PostgreSQL::Test::Cluster - if ($PG_MAJOR_VERSION > 14) { - $pg_node = PostgreSQL::Test::Cluster->new('pgtde_regression'); - } - # For Server version 14 & below, spawn the server using PostgresNode - elsif ($PG_MAJOR_VERSION < 15) { - $pg_node = PostgresNode->get_new_node('pgtde_regression'); - } - - $pg_node->dump_info; - $pg_node->init; - return $pg_node; + my ($node, $dbname, $sql) = @_; + + my (undef, $stdout, $stderr) = $node->psql($dbname, $sql, + extra_params => [ '-a', '-Pformat=aligned', '-Ptuples_only=off' ]); + + if ($stdout ne '') + { + append_to_result_file($stdout); + } + + if ($stderr ne '') + { + append_to_result_file($stderr); + } } -sub append_to_file +sub append_to_result_file { - my ($str) = @_; - - # For Server version 15 & above, use PostgreSQL::Test::Utils to write to files - if ($PG_MAJOR_VERSION > 14) { - PostgreSQL::Test::Utils::append_to_file($out_filename_with_path, $str . "\n"); - } - # For Server version 14 & below, use PostgresNode to write to files - elsif ($PG_MAJOR_VERSION < 15) { - TestLib::append_to_file($out_filename_with_path, $str . "\n"); - } - chmod(0640 , $out_filename_with_path) - or die("unable to set permissions for $out_filename_with_path"); - - return; + my ($str) = @_; + + append_to_file($out_filename_with_path, $str . "\n"); } sub append_to_debug_file { - my ($str) = @_; - - # For Server version 15 & above, use PostgreSQL::Test::Utils to write to files - if ($PG_MAJOR_VERSION > 14) { - PostgreSQL::Test::Utils::append_to_file($debug_out_filename_with_path, $str . "\n"); - } - # For Server version 14 & below, use PostgresNode to write to files - elsif ($PG_MAJOR_VERSION < 15) { - TestLib::append_to_file($debug_out_filename_with_path, $str . "\n"); - } - chmod(0640 , $debug_out_filename_with_path) - or die("unable to set permissions for $debug_out_filename_with_path"); - - return; + my ($str) = @_; + + append_to_file($debug_out_filename_with_path, $str . "\n"); } sub setup_files_dir { - my ($perlfilename) = @_; - - # Expected folder where expected output will be present - my $expected_folder = "t/expected"; - - # Results/out folder where generated results files will be placed - my $results_folder = "t/results"; - - # Check if results folder exists or not, create if it doesn't - unless (-d $results_folder) - { - mkdir $results_folder or die "Can't create folder $results_folder: $!\n"; - } - - # Check if expected folder exists or not, bail out if it doesn't - unless (-d $expected_folder) - { - BAIL_OUT "Expected files folder $expected_folder doesn't exist: \n"; - } - - #Remove .pl from filename and store in a variable - my @split_arr = split /\./, $perlfilename; - - my $filename_without_extension = $split_arr[0]; - - # Create expected filename with path - my $expected_filename = "${filename_without_extension}.out"; - - $expected_filename_with_path = "${expected_folder}/${expected_filename}"; - - # Create results filename with path - my $out_filename = "${filename_without_extension}.out"; - $out_filename_with_path = "${results_folder}/${out_filename}"; - - # Delete already existing result out file, if it exists. - if ( -f $out_filename_with_path) - { - unlink($out_filename_with_path) or die "Can't delete already existing $out_filename_with_path: $!\n"; - } - - $debug_out_filename_with_path = "${results_folder}/${out_filename}.debug"; + my ($test_filename) = @_; + + unless (-d $results_folder) + { + mkdir $results_folder + or die "Can't create folder $results_folder: $!\n"; + } + + my ($test_name) = $test_filename =~ /([^.]*)/; + + $expected_filename_with_path = "${expected_folder}/${test_name}.out"; + $out_filename_with_path = "${results_folder}/${test_name}.out"; + $debug_out_filename_with_path = + "${results_folder}/${test_name}.out.debug"; + + if (-f $out_filename_with_path) + { + unlink($out_filename_with_path) + or die + "Can't delete already existing $out_filename_with_path: $!\n"; + } } sub compare_results { - # Compare expected and results files and return the result - return compare($expected_filename_with_path, $out_filename_with_path); + return compare($expected_filename_with_path, $out_filename_with_path); } 1; diff --git a/contrib/pg_tde/typedefs.list b/contrib/pg_tde/typedefs.list deleted file mode 100644 index 7ff41d54e5fd9..0000000000000 --- a/contrib/pg_tde/typedefs.list +++ /dev/null @@ -1,110 +0,0 @@ -BulkInsertStateData -BulkInsertStateData -BulkInsertStateData -BulkInsertStateData -CurlString -FileKeyring -GenericKeyring -HeapPageFreeze -HeapPageFreeze -HeapScanDescData -HeapScanDescData -HeapScanDescData -HeapScanDescData -HeapTupleFreeze -HeapTupleFreeze -IndexDeleteCounts -IndexDeleteCounts -IndexFetchHeapData -IndexFetchHeapData -InternalKey -JsonKeringSemState -JsonKeyringField -JsonKeyringState -JsonVaultRespField -JsonVaultRespSemState -JsonVaultRespState -Keydata -KeyInfo -KeyProviders -KeyringProvideRecord -KeyringProviderXLRecord -KeyringReturnCode -LVPagePruneState -LVRelState -LVRelState -LVSavedErrInfo -LVSavedErrInfo -LogicalRewriteMappingData -LogicalRewriteMappingData -PendingMapEntryDelete -ProviderScanType -PruneFreezeResult -RelKeyCache -RelKeyCacheRec -RewriteMappingDataEntry -RewriteMappingDataEntry -RewriteMappingFile -RewriteMappingFile -RewriteStateData -RewriteStateData -RewriteStateData -RewriteStateData *RewriteState; -TDEBufferHeapTupleTableSlot -TDEFileHeader -TDEKeyringRoutine -TDELocalState -TDELockTypes -TDEMapEntry -TDEMapFilePath -TDEPrincipalKey -TDEPrincipalKeyInfo -TDEShmemSetupRoutine -TdeCreateEvent -TdeCreateEventType -TdeKeyProviderInfoSharedState -TdePrincipalKeySharedState -TdePrincipalKeylocalState -TdeSharedState -VaultV2Keyring -XLogExtensionInstall -XLogPrincipalKeyRotate -XLogRelKey -itemIdCompactData -itemIdCompactData -xl_multi_insert_tuple -xl_multi_insert_tuple -xl_tdeheap_confirm -xl_tdeheap_confirm -xl_tdeheap_delete -xl_tdeheap_delete -xl_tdeheap_freeze_page -xl_tdeheap_freeze_plan -xl_tdeheap_header -xl_tdeheap_header -xl_tdeheap_inplace -xl_tdeheap_inplace -xl_tdeheap_insert -xl_tdeheap_insert -xl_tdeheap_lock -xl_tdeheap_lock -xl_tdeheap_lock_updated -xl_tdeheap_lock_updated -xl_tdeheap_multi_insert -xl_tdeheap_multi_insert -xl_tdeheap_new_cid -xl_tdeheap_new_cid -xl_tdeheap_prune -xl_tdeheap_prune -xl_tdeheap_rewrite_mapping -xl_tdeheap_rewrite_mapping -xl_tdeheap_truncate -xl_tdeheap_truncate -xl_tdeheap_update -xl_tdeheap_update -xl_tdeheap_vacuum -xl_tdeheap_visible -xl_tdeheap_visible -xlhp_freeze_plan -xlhp_freeze_plans -xlhp_prune_items \ No newline at end of file diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index e1b85132eef49..833b07a970971 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -138,7 +138,7 @@ pg_tde_init(const char *datadir) static bool is_pg_tde_encypted(Oid spcOid, Oid dbOid, RelFileNumber relNumber) { - RelFileLocator locator = {.spcOid = spcOid, dbOid = dbOid,.relNumber = relNumber}; + RelFileLocator locator = {.spcOid = spcOid, .dbOid = dbOid,.relNumber = relNumber}; RelFileLocatorBackend rlocator = {.locator = locator,.backend = INVALID_PROC_NUMBER}; return GetSMGRRelationKey(rlocator) != NULL; diff --git a/src/bin/pg_tde_change_key_provider/.gitignore b/src/bin/pg_tde_change_key_provider/.gitignore deleted file mode 100644 index 028680dcbc3e8..0000000000000 --- a/src/bin/pg_tde_change_key_provider/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/pg_tde_change_key_provider diff --git a/src/bin/pg_waldump/meson.build b/src/bin/pg_waldump/meson.build index cab9de90846f2..2d9d67a02ed82 100644 --- a/src/bin/pg_waldump/meson.build +++ b/src/bin/pg_waldump/meson.build @@ -42,8 +42,6 @@ tests += { 'tests': [ 't/001_basic.pl', 't/002_save_fullpage.pl', - 't/003_basic_encrypted.pl', - 't/004_save_fullpage_encrypted.pl', ], }, } diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 0ee07d33f4fa3..171d6a52955fa 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -34,7 +34,7 @@ #ifdef PERCONA_EXT #include "access/pg_tde_fe_init.h" -#include "access/pg_tde_xlog_encrypt.h" +#include "access/pg_tde_xlog_smgr.h" #include "access/xlog_smgr.h" #include "catalog/tde_global_space.h" #endif diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 0ed589426af86..67c90a2bd328d 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -945,7 +945,7 @@ typedef struct PartitionRangeDatum typedef struct SinglePartitionSpec { NodeTag type; -} SinglePartitionSpec; +} SinglePartitionSpec; /* * PartitionCmd - info for ALTER TABLE/INDEX ATTACH/DETACH PARTITION commands diff --git a/src/include/utils/percona.h b/src/include/utils/percona.h index 87bad00df784a..e8de53618c551 100644 --- a/src/include/utils/percona.h +++ b/src/include/utils/percona.h @@ -14,16 +14,13 @@ extern const PGDLLIMPORT int percona_api_version; -static inline bool +static inline void check_percona_api_version(void) { if (PERCONA_API_VERSION != percona_api_version) { elog(FATAL, "Percona API version mismatch, the extension was built against a different PostgreSQL version!"); - return false; } - - return true; } #endif diff --git a/src/test/modules/test_misc/t/008_percona_server_version.pl b/src/test/modules/test_misc/t/008_percona_server_version.pl new file mode 100644 index 0000000000000..c7723525ebf22 --- /dev/null +++ b/src/test/modules/test_misc/t/008_percona_server_version.pl @@ -0,0 +1,62 @@ +#!/usr/bin/perl +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use lib 't'; +use Env; + +plan tests => 6; + +# Initialize a test cluster +my $node = PostgreSQL::Test::Cluster->new('pg_server'); +$node->init(); +my $pgdata = $node->data_dir; + +# To make this testcase work, PERCONA_SERVER_VERSION variable should be available in environment. +# If you are using ci_scripts it is already declated in ci_scripts/env.sh +# If you are using command line make for regression then export like: +# export PERCONA_SERVER_VERSION=17.4.1 + +if (!defined($ENV{PERCONA_SERVER_VERSION})) +{ + BAIL_OUT("PERCONA_SERVER_VERSION variable not present in the environment"); +} + +my $percona_expected_server_version = $ENV{PERCONA_SERVER_VERSION}; + +# Start server +my $rt_value = $node->start; +ok($rt_value == 1, "Start Server"); + +# Get PG Server version (e.g 17.4) from pg_config +my $pg_server_version = `pg_config --version | awk {'print \$2'}`; +$pg_server_version=~ s/^\s+|\s+$//g; + +# Check pg_config output. +my $pg_config_output = `pg_config --version`; +$pg_config_output=~ s/^\s+|\s+$//g; +cmp_ok($pg_config_output,'eq',"PostgreSQL $pg_server_version - Percona Server for PostgreSQL $percona_expected_server_version", "Test pg_config --version output"); + +# Check psql --version output. +my $psql_version_output = `psql --version`; +$psql_version_output=~ s/^\s+|\s+$//g; +cmp_ok($psql_version_output,'eq',"psql (PostgreSQL) $pg_server_version - Percona Server for PostgreSQL $percona_expected_server_version", "Test psql --version output"); + +# Check postgres --version output. +my $postgres_output = `postgres --version`; +$postgres_output=~ s/^\s+|\s+$//g; +cmp_ok($postgres_output,'eq',"postgres (PostgreSQL) $pg_server_version - Percona Server for PostgreSQL $percona_expected_server_version", "Test postgres --version output"); + +# Check select version() output. +my ($cmdret, $stdout, $stderr) = $node->psql('postgres', "select version();", extra_params => ['-a', '-Pformat=aligned','-Ptuples_only=on']); +ok($cmdret == 0, "# Get output of select version();"); +$stdout=~ s/^\s+|\s+$//g; +like($stdout, "/PostgreSQL $pg_server_version - Percona Server for PostgreSQL $percona_expected_server_version/", "Test select version() output"); + +# Stop the server +$node->stop; + +# Done testing for this testcase file. +done_testing(); diff --git a/src/test/regress/expected/create_index_2.out b/src/test/regress/expected/create_index_2.out deleted file mode 100644 index f29407adc69c9..0000000000000 --- a/src/test/regress/expected/create_index_2.out +++ /dev/null @@ -1,3000 +0,0 @@ --- --- CREATE_INDEX --- Create ancillary data structures (i.e. indices) --- --- directory paths are passed to us in environment variables -\getenv abs_srcdir PG_ABS_SRCDIR --- --- BTREE --- -CREATE INDEX onek_unique1 ON onek USING btree(unique1 int4_ops); -CREATE INDEX IF NOT EXISTS onek_unique1 ON onek USING btree(unique1 int4_ops); -NOTICE: relation "onek_unique1" already exists, skipping -CREATE INDEX IF NOT EXISTS ON onek USING btree(unique1 int4_ops); -ERROR: syntax error at or near "ON" -LINE 1: CREATE INDEX IF NOT EXISTS ON onek USING btree(unique1 int4_... - ^ -CREATE INDEX onek_unique2 ON onek USING btree(unique2 int4_ops); -CREATE INDEX onek_hundred ON onek USING btree(hundred int4_ops); -CREATE INDEX onek_stringu1 ON onek USING btree(stringu1 name_ops); -CREATE INDEX tenk1_unique1 ON tenk1 USING btree(unique1 int4_ops); -CREATE INDEX tenk1_unique2 ON tenk1 USING btree(unique2 int4_ops); -CREATE INDEX tenk1_hundred ON tenk1 USING btree(hundred int4_ops); -CREATE INDEX tenk1_thous_tenthous ON tenk1 (thousand, tenthous); -CREATE INDEX tenk2_unique1 ON tenk2 USING btree(unique1 int4_ops); -CREATE INDEX tenk2_unique2 ON tenk2 USING btree(unique2 int4_ops); -CREATE INDEX tenk2_hundred ON tenk2 USING btree(hundred int4_ops); -CREATE INDEX rix ON road USING btree (name text_ops); -CREATE INDEX iix ON ihighway USING btree (name text_ops); -CREATE INDEX six ON shighway USING btree (name text_ops); --- test comments -COMMENT ON INDEX six_wrong IS 'bad index'; -ERROR: relation "six_wrong" does not exist -COMMENT ON INDEX six IS 'good index'; -COMMENT ON INDEX six IS NULL; --- --- BTREE partial indices --- -CREATE INDEX onek2_u1_prtl ON onek2 USING btree(unique1 int4_ops) - where unique1 < 20 or unique1 > 980; -CREATE INDEX onek2_u2_prtl ON onek2 USING btree(unique2 int4_ops) - where stringu1 < 'B'; -CREATE INDEX onek2_stu1_prtl ON onek2 USING btree(stringu1 name_ops) - where onek2.stringu1 >= 'J' and onek2.stringu1 < 'K'; --- --- GiST (rtree-equivalent opclasses only) --- -CREATE TABLE slow_emp4000 ( - home_base box -); -CREATE TABLE fast_emp4000 ( - home_base box -); -\set filename :abs_srcdir '/data/rect.data' -COPY slow_emp4000 FROM :'filename'; -INSERT INTO fast_emp4000 SELECT * FROM slow_emp4000; -ANALYZE slow_emp4000; -ANALYZE fast_emp4000; -CREATE INDEX grect2ind ON fast_emp4000 USING gist (home_base); --- we want to work with a point_tbl that includes a null -CREATE TEMP TABLE point_tbl AS SELECT * FROM public.point_tbl; -INSERT INTO POINT_TBL(f1) VALUES (NULL); -CREATE INDEX gpointind ON point_tbl USING gist (f1); -CREATE TEMP TABLE gpolygon_tbl AS - SELECT polygon(home_base) AS f1 FROM slow_emp4000; -INSERT INTO gpolygon_tbl VALUES ( '(1000,0,0,1000)' ); -INSERT INTO gpolygon_tbl VALUES ( '(0,1000,1000,1000)' ); -CREATE TEMP TABLE gcircle_tbl AS - SELECT circle(home_base) AS f1 FROM slow_emp4000; -CREATE INDEX ggpolygonind ON gpolygon_tbl USING gist (f1); -CREATE INDEX ggcircleind ON gcircle_tbl USING gist (f1); --- --- Test GiST indexes --- --- get non-indexed results for comparison purposes -SET enable_seqscan = ON; -SET enable_indexscan = OFF; -SET enable_bitmapscan = OFF; -SELECT * FROM fast_emp4000 - WHERE home_base <@ '(200,200),(2000,1000)'::box - ORDER BY (home_base[0])[0]; - home_base ------------------------ - (337,455),(240,359) - (1444,403),(1346,344) -(2 rows) - -SELECT count(*) FROM fast_emp4000 WHERE home_base && '(1000,1000,0,0)'::box; - count -------- - 2 -(1 row) - -SELECT count(*) FROM fast_emp4000 WHERE home_base IS NULL; - count -------- - 278 -(1 row) - -SELECT count(*) FROM gpolygon_tbl WHERE f1 && '(1000,1000,0,0)'::polygon; - count -------- - 2 -(1 row) - -SELECT count(*) FROM gcircle_tbl WHERE f1 && '<(500,500),500>'::circle; - count -------- - 2 -(1 row) - -SELECT count(*) FROM point_tbl WHERE f1 <@ box '(0,0,100,100)'; - count -------- - 3 -(1 row) - -SELECT count(*) FROM point_tbl WHERE box '(0,0,100,100)' @> f1; - count -------- - 3 -(1 row) - -SELECT count(*) FROM point_tbl WHERE f1 <@ polygon '(0,0),(0,100),(100,100),(50,50),(100,0),(0,0)'; - count -------- - 5 -(1 row) - -SELECT count(*) FROM point_tbl WHERE f1 <@ circle '<(50,50),50>'; - count -------- - 1 -(1 row) - -SELECT count(*) FROM point_tbl p WHERE p.f1 << '(0.0, 0.0)'; - count -------- - 3 -(1 row) - -SELECT count(*) FROM point_tbl p WHERE p.f1 >> '(0.0, 0.0)'; - count -------- - 4 -(1 row) - -SELECT count(*) FROM point_tbl p WHERE p.f1 <<| '(0.0, 0.0)'; - count -------- - 1 -(1 row) - -SELECT count(*) FROM point_tbl p WHERE p.f1 |>> '(0.0, 0.0)'; - count -------- - 5 -(1 row) - -SELECT count(*) FROM point_tbl p WHERE p.f1 ~= '(-5, -12)'; - count -------- - 1 -(1 row) - -SELECT * FROM point_tbl ORDER BY f1 <-> '0,1'; - f1 -------------------- - (0,0) - (1e-300,-1e-300) - (-3,4) - (-10,0) - (10,10) - (-5,-12) - (5.1,34.5) - (Infinity,1e+300) - (1e+300,Infinity) - (NaN,NaN) - -(11 rows) - -SELECT * FROM point_tbl WHERE f1 IS NULL; - f1 ----- - -(1 row) - -SELECT * FROM point_tbl WHERE f1 IS NOT NULL ORDER BY f1 <-> '0,1'; - f1 -------------------- - (0,0) - (1e-300,-1e-300) - (-3,4) - (-10,0) - (10,10) - (-5,-12) - (5.1,34.5) - (1e+300,Infinity) - (Infinity,1e+300) - (NaN,NaN) -(10 rows) - -SELECT * FROM point_tbl WHERE f1 <@ '(-10,-10),(10,10)':: box ORDER BY f1 <-> '0,1'; - f1 ------------------- - (0,0) - (1e-300,-1e-300) - (-3,4) - (-10,0) - (10,10) -(5 rows) - -SELECT * FROM gpolygon_tbl ORDER BY f1 <-> '(0,0)'::point LIMIT 10; - f1 -------------------------------------------------- - ((240,359),(240,455),(337,455),(337,359)) - ((662,163),(662,187),(759,187),(759,163)) - ((1000,0),(0,1000)) - ((0,1000),(1000,1000)) - ((1346,344),(1346,403),(1444,403),(1444,344)) - ((278,1409),(278,1457),(369,1457),(369,1409)) - ((907,1156),(907,1201),(948,1201),(948,1156)) - ((1517,971),(1517,1043),(1594,1043),(1594,971)) - ((175,1820),(175,1850),(259,1850),(259,1820)) - ((2424,81),(2424,160),(2424,160),(2424,81)) -(10 rows) - -SELECT circle_center(f1), round(radius(f1)) as radius FROM gcircle_tbl ORDER BY f1 <-> '(200,300)'::point LIMIT 10; - circle_center | radius -----------------+-------- - (288.5,407) | 68 - (710.5,175) | 50 - (323.5,1433) | 51 - (927.5,1178.5) | 30 - (1395,373.5) | 57 - (1555.5,1007) | 53 - (217,1835) | 45 - (489,2421.5) | 22 - (2424,120.5) | 40 - (751.5,2655) | 20 -(10 rows) - --- Now check the results from plain indexscan -SET enable_seqscan = OFF; -SET enable_indexscan = ON; -SET enable_bitmapscan = OFF; -EXPLAIN (COSTS OFF) -SELECT * FROM fast_emp4000 - WHERE home_base <@ '(200,200),(2000,1000)'::box - ORDER BY (home_base[0])[0]; - QUERY PLAN ------------------------------------------------------------------ - Sort - Sort Key: ((home_base[0])[0]) - -> Index Only Scan using grect2ind on fast_emp4000 - Index Cond: (home_base <@ '(2000,1000),(200,200)'::box) -(4 rows) - -SELECT * FROM fast_emp4000 - WHERE home_base <@ '(200,200),(2000,1000)'::box - ORDER BY (home_base[0])[0]; - home_base ------------------------ - (337,455),(240,359) - (1444,403),(1346,344) -(2 rows) - -EXPLAIN (COSTS OFF) -SELECT count(*) FROM fast_emp4000 WHERE home_base && '(1000,1000,0,0)'::box; - QUERY PLAN -------------------------------------------------------------- - Aggregate - -> Index Only Scan using grect2ind on fast_emp4000 - Index Cond: (home_base && '(1000,1000),(0,0)'::box) -(3 rows) - -SELECT count(*) FROM fast_emp4000 WHERE home_base && '(1000,1000,0,0)'::box; - count -------- - 2 -(1 row) - -EXPLAIN (COSTS OFF) -SELECT count(*) FROM fast_emp4000 WHERE home_base IS NULL; - QUERY PLAN -------------------------------------------------------- - Aggregate - -> Index Only Scan using grect2ind on fast_emp4000 - Index Cond: (home_base IS NULL) -(3 rows) - -SELECT count(*) FROM fast_emp4000 WHERE home_base IS NULL; - count -------- - 278 -(1 row) - -EXPLAIN (COSTS OFF) -SELECT count(*) FROM gpolygon_tbl WHERE f1 && '(1000,1000,0,0)'::polygon; - QUERY PLAN ------------------------------------------------------------- - Aggregate - -> Index Scan using ggpolygonind on gpolygon_tbl - Index Cond: (f1 && '((1000,1000),(0,0))'::polygon) -(3 rows) - -SELECT count(*) FROM gpolygon_tbl WHERE f1 && '(1000,1000,0,0)'::polygon; - count -------- - 2 -(1 row) - -EXPLAIN (COSTS OFF) -SELECT count(*) FROM gcircle_tbl WHERE f1 && '<(500,500),500>'::circle; - QUERY PLAN -------------------------------------------------------- - Aggregate - -> Index Scan using ggcircleind on gcircle_tbl - Index Cond: (f1 && '<(500,500),500>'::circle) -(3 rows) - -SELECT count(*) FROM gcircle_tbl WHERE f1 && '<(500,500),500>'::circle; - count -------- - 2 -(1 row) - -EXPLAIN (COSTS OFF) -SELECT count(*) FROM point_tbl WHERE f1 <@ box '(0,0,100,100)'; - QUERY PLAN ----------------------------------------------------- - Aggregate - -> Index Only Scan using gpointind on point_tbl - Index Cond: (f1 <@ '(100,100),(0,0)'::box) -(3 rows) - -SELECT count(*) FROM point_tbl WHERE f1 <@ box '(0,0,100,100)'; - count -------- - 3 -(1 row) - -EXPLAIN (COSTS OFF) -SELECT count(*) FROM point_tbl WHERE box '(0,0,100,100)' @> f1; - QUERY PLAN ----------------------------------------------------- - Aggregate - -> Index Only Scan using gpointind on point_tbl - Index Cond: (f1 <@ '(100,100),(0,0)'::box) -(3 rows) - -SELECT count(*) FROM point_tbl WHERE box '(0,0,100,100)' @> f1; - count -------- - 3 -(1 row) - -EXPLAIN (COSTS OFF) -SELECT count(*) FROM point_tbl WHERE f1 <@ polygon '(0,0),(0,100),(100,100),(50,50),(100,0),(0,0)'; - QUERY PLAN ----------------------------------------------------------------------------------------- - Aggregate - -> Index Only Scan using gpointind on point_tbl - Index Cond: (f1 <@ '((0,0),(0,100),(100,100),(50,50),(100,0),(0,0))'::polygon) -(3 rows) - -SELECT count(*) FROM point_tbl WHERE f1 <@ polygon '(0,0),(0,100),(100,100),(50,50),(100,0),(0,0)'; - count -------- - 4 -(1 row) - -EXPLAIN (COSTS OFF) -SELECT count(*) FROM point_tbl WHERE f1 <@ circle '<(50,50),50>'; - QUERY PLAN ----------------------------------------------------- - Aggregate - -> Index Only Scan using gpointind on point_tbl - Index Cond: (f1 <@ '<(50,50),50>'::circle) -(3 rows) - -SELECT count(*) FROM point_tbl WHERE f1 <@ circle '<(50,50),50>'; - count -------- - 1 -(1 row) - -EXPLAIN (COSTS OFF) -SELECT count(*) FROM point_tbl p WHERE p.f1 << '(0.0, 0.0)'; - QUERY PLAN ------------------------------------------------------- - Aggregate - -> Index Only Scan using gpointind on point_tbl p - Index Cond: (f1 << '(0,0)'::point) -(3 rows) - -SELECT count(*) FROM point_tbl p WHERE p.f1 << '(0.0, 0.0)'; - count -------- - 3 -(1 row) - -EXPLAIN (COSTS OFF) -SELECT count(*) FROM point_tbl p WHERE p.f1 >> '(0.0, 0.0)'; - QUERY PLAN ------------------------------------------------------- - Aggregate - -> Index Only Scan using gpointind on point_tbl p - Index Cond: (f1 >> '(0,0)'::point) -(3 rows) - -SELECT count(*) FROM point_tbl p WHERE p.f1 >> '(0.0, 0.0)'; - count -------- - 4 -(1 row) - -EXPLAIN (COSTS OFF) -SELECT count(*) FROM point_tbl p WHERE p.f1 <<| '(0.0, 0.0)'; - QUERY PLAN ------------------------------------------------------- - Aggregate - -> Index Only Scan using gpointind on point_tbl p - Index Cond: (f1 <<| '(0,0)'::point) -(3 rows) - -SELECT count(*) FROM point_tbl p WHERE p.f1 <<| '(0.0, 0.0)'; - count -------- - 1 -(1 row) - -EXPLAIN (COSTS OFF) -SELECT count(*) FROM point_tbl p WHERE p.f1 |>> '(0.0, 0.0)'; - QUERY PLAN ------------------------------------------------------- - Aggregate - -> Index Only Scan using gpointind on point_tbl p - Index Cond: (f1 |>> '(0,0)'::point) -(3 rows) - -SELECT count(*) FROM point_tbl p WHERE p.f1 |>> '(0.0, 0.0)'; - count -------- - 5 -(1 row) - -EXPLAIN (COSTS OFF) -SELECT count(*) FROM point_tbl p WHERE p.f1 ~= '(-5, -12)'; - QUERY PLAN ------------------------------------------------------- - Aggregate - -> Index Only Scan using gpointind on point_tbl p - Index Cond: (f1 ~= '(-5,-12)'::point) -(3 rows) - -SELECT count(*) FROM point_tbl p WHERE p.f1 ~= '(-5, -12)'; - count -------- - 1 -(1 row) - -EXPLAIN (COSTS OFF) -SELECT * FROM point_tbl ORDER BY f1 <-> '0,1'; - QUERY PLAN ----------------------------------------------- - Index Only Scan using gpointind on point_tbl - Order By: (f1 <-> '(0,1)'::point) -(2 rows) - -SELECT * FROM point_tbl ORDER BY f1 <-> '0,1'; - f1 -------------------- - (1e-300,-1e-300) - (0,0) - (-3,4) - (-10,0) - (10,10) - (-5,-12) - (5.1,34.5) - (Infinity,1e+300) - (1e+300,Infinity) - (NaN,NaN) - -(11 rows) - -EXPLAIN (COSTS OFF) -SELECT * FROM point_tbl WHERE f1 IS NULL; - QUERY PLAN ----------------------------------------------- - Index Only Scan using gpointind on point_tbl - Index Cond: (f1 IS NULL) -(2 rows) - -SELECT * FROM point_tbl WHERE f1 IS NULL; - f1 ----- - -(1 row) - -EXPLAIN (COSTS OFF) -SELECT * FROM point_tbl WHERE f1 IS NOT NULL ORDER BY f1 <-> '0,1'; - QUERY PLAN ----------------------------------------------- - Index Only Scan using gpointind on point_tbl - Index Cond: (f1 IS NOT NULL) - Order By: (f1 <-> '(0,1)'::point) -(3 rows) - -SELECT * FROM point_tbl WHERE f1 IS NOT NULL ORDER BY f1 <-> '0,1'; - f1 -------------------- - (1e-300,-1e-300) - (0,0) - (-3,4) - (-10,0) - (10,10) - (-5,-12) - (5.1,34.5) - (Infinity,1e+300) - (1e+300,Infinity) - (NaN,NaN) -(10 rows) - -EXPLAIN (COSTS OFF) -SELECT * FROM point_tbl WHERE f1 <@ '(-10,-10),(10,10)':: box ORDER BY f1 <-> '0,1'; - QUERY PLAN ------------------------------------------------- - Index Only Scan using gpointind on point_tbl - Index Cond: (f1 <@ '(10,10),(-10,-10)'::box) - Order By: (f1 <-> '(0,1)'::point) -(3 rows) - -SELECT * FROM point_tbl WHERE f1 <@ '(-10,-10),(10,10)':: box ORDER BY f1 <-> '0,1'; - f1 ------------------- - (1e-300,-1e-300) - (0,0) - (-3,4) - (-10,0) - (10,10) -(5 rows) - -EXPLAIN (COSTS OFF) -SELECT * FROM gpolygon_tbl ORDER BY f1 <-> '(0,0)'::point LIMIT 10; - QUERY PLAN ------------------------------------------------------ - Limit - -> Index Scan using ggpolygonind on gpolygon_tbl - Order By: (f1 <-> '(0,0)'::point) -(3 rows) - -SELECT * FROM gpolygon_tbl ORDER BY f1 <-> '(0,0)'::point LIMIT 10; - f1 -------------------------------------------------- - ((240,359),(240,455),(337,455),(337,359)) - ((662,163),(662,187),(759,187),(759,163)) - ((1000,0),(0,1000)) - ((0,1000),(1000,1000)) - ((1346,344),(1346,403),(1444,403),(1444,344)) - ((278,1409),(278,1457),(369,1457),(369,1409)) - ((907,1156),(907,1201),(948,1201),(948,1156)) - ((1517,971),(1517,1043),(1594,1043),(1594,971)) - ((175,1820),(175,1850),(259,1850),(259,1820)) - ((2424,81),(2424,160),(2424,160),(2424,81)) -(10 rows) - -EXPLAIN (COSTS OFF) -SELECT circle_center(f1), round(radius(f1)) as radius FROM gcircle_tbl ORDER BY f1 <-> '(200,300)'::point LIMIT 10; - QUERY PLAN ---------------------------------------------------- - Limit - -> Index Scan using ggcircleind on gcircle_tbl - Order By: (f1 <-> '(200,300)'::point) -(3 rows) - -SELECT circle_center(f1), round(radius(f1)) as radius FROM gcircle_tbl ORDER BY f1 <-> '(200,300)'::point LIMIT 10; - circle_center | radius -----------------+-------- - (288.5,407) | 68 - (710.5,175) | 50 - (323.5,1433) | 51 - (927.5,1178.5) | 30 - (1395,373.5) | 57 - (1555.5,1007) | 53 - (217,1835) | 45 - (489,2421.5) | 22 - (2424,120.5) | 40 - (751.5,2655) | 20 -(10 rows) - -EXPLAIN (COSTS OFF) -SELECT point(x,x), (SELECT f1 FROM gpolygon_tbl ORDER BY f1 <-> point(x,x) LIMIT 1) as c FROM generate_series(0,10,1) x; - QUERY PLAN --------------------------------------------------------------------------------------------- - Function Scan on generate_series x - SubPlan 1 - -> Limit - -> Index Scan using ggpolygonind on gpolygon_tbl - Order By: (f1 <-> point((x.x)::double precision, (x.x)::double precision)) -(5 rows) - -SELECT point(x,x), (SELECT f1 FROM gpolygon_tbl ORDER BY f1 <-> point(x,x) LIMIT 1) as c FROM generate_series(0,10,1) x; - point | c ----------+------------------------------------------- - (0,0) | ((240,359),(240,455),(337,455),(337,359)) - (1,1) | ((240,359),(240,455),(337,455),(337,359)) - (2,2) | ((240,359),(240,455),(337,455),(337,359)) - (3,3) | ((240,359),(240,455),(337,455),(337,359)) - (4,4) | ((240,359),(240,455),(337,455),(337,359)) - (5,5) | ((240,359),(240,455),(337,455),(337,359)) - (6,6) | ((240,359),(240,455),(337,455),(337,359)) - (7,7) | ((240,359),(240,455),(337,455),(337,359)) - (8,8) | ((240,359),(240,455),(337,455),(337,359)) - (9,9) | ((240,359),(240,455),(337,455),(337,359)) - (10,10) | ((240,359),(240,455),(337,455),(337,359)) -(11 rows) - --- Now check the results from bitmap indexscan -SET enable_seqscan = OFF; -SET enable_indexscan = OFF; -SET enable_bitmapscan = ON; -EXPLAIN (COSTS OFF) -SELECT * FROM point_tbl WHERE f1 <@ '(-10,-10),(10,10)':: box ORDER BY f1 <-> '0,1'; - QUERY PLAN ------------------------------------------------------------- - Sort - Sort Key: ((f1 <-> '(0,1)'::point)) - -> Bitmap Heap Scan on point_tbl - Recheck Cond: (f1 <@ '(10,10),(-10,-10)'::box) - -> Bitmap Index Scan on gpointind - Index Cond: (f1 <@ '(10,10),(-10,-10)'::box) -(6 rows) - -SELECT * FROM point_tbl WHERE f1 <@ '(-10,-10),(10,10)':: box ORDER BY f1 <-> '0,1'; - f1 ------------------- - (0,0) - (1e-300,-1e-300) - (-3,4) - (-10,0) - (10,10) -(5 rows) - -RESET enable_seqscan; -RESET enable_indexscan; -RESET enable_bitmapscan; --- --- GIN over int[] and text[] --- --- Note: GIN currently supports only bitmap scans, not plain indexscans --- -CREATE TABLE array_index_op_test ( - seqno int4, - i int4[], - t text[] -); -\set filename :abs_srcdir '/data/array.data' -COPY array_index_op_test FROM :'filename'; -ANALYZE array_index_op_test; -SELECT * FROM array_index_op_test WHERE i = '{NULL}' ORDER BY seqno; - seqno | i | t --------+--------+-------- - 102 | {NULL} | {NULL} -(1 row) - -SELECT * FROM array_index_op_test WHERE i @> '{NULL}' ORDER BY seqno; - seqno | i | t --------+---+--- -(0 rows) - -SELECT * FROM array_index_op_test WHERE i && '{NULL}' ORDER BY seqno; - seqno | i | t --------+---+--- -(0 rows) - -SELECT * FROM array_index_op_test WHERE i <@ '{NULL}' ORDER BY seqno; - seqno | i | t --------+----+---- - 101 | {} | {} -(1 row) - -SET enable_seqscan = OFF; -SET enable_indexscan = OFF; -SET enable_bitmapscan = ON; -CREATE INDEX intarrayidx ON array_index_op_test USING gin (i); -explain (costs off) -SELECT * FROM array_index_op_test WHERE i @> '{32}' ORDER BY seqno; - QUERY PLAN ----------------------------------------------------- - Sort - Sort Key: seqno - -> Bitmap Heap Scan on array_index_op_test - Recheck Cond: (i @> '{32}'::integer[]) - -> Bitmap Index Scan on intarrayidx - Index Cond: (i @> '{32}'::integer[]) -(6 rows) - -SELECT * FROM array_index_op_test WHERE i @> '{32}' ORDER BY seqno; - seqno | i | t --------+---------------------------------+------------------------------------------------------------------------------------------------------------------------------------ - 6 | {39,35,5,94,17,92,60,32} | {AAAAAAAAAAAAAAA35875,AAAAAAAAAAAAAAAA23657} - 74 | {32} | {AAAAAAAAAAAAAAAA1729,AAAAAAAAAAAAA22860,AAAAAA99807,AAAAA17383,AAAAAAAAAAAAAAA67062,AAAAAAAAAAA15165,AAAAAAAAAAA50956} - 77 | {97,15,32,17,55,59,18,37,50,39} | {AAAAAAAAAAAA67946,AAAAAA54032,AAAAAAAA81587,55847,AAAAAAAAAAAAAA28620,AAAAAAAAAAAAAAAAA43052,AAAAAA75463,AAAA49534,AAAAAAAA44066} - 89 | {40,32,17,6,30,88} | {AA44673,AAAAAAAAAAA6119,AAAAAAAAAAAAAAAA23657,AAAAAAAAAAAAAAAAAA47955,AAAAAAAAAAAAAAAA33598,AAAAAAAAAAA33576,AA44673} - 98 | {38,34,32,89} | {AAAAAAAAAAAAAAAAAA71621,AAAA8857,AAAAAAAAAAAAAAAAAAA65037,AAAAAAAAAAAAAAAA31334,AAAAAAAAAA48845} - 100 | {85,32,57,39,49,84,32,3,30} | {AAAAAAA80240,AAAAAAAAAAAAAAAA1729,AAAAA60038,AAAAAAAAAAA92631,AAAAAAAA9523} -(6 rows) - -SELECT * FROM array_index_op_test WHERE i && '{32}' ORDER BY seqno; - seqno | i | t --------+---------------------------------+------------------------------------------------------------------------------------------------------------------------------------ - 6 | {39,35,5,94,17,92,60,32} | {AAAAAAAAAAAAAAA35875,AAAAAAAAAAAAAAAA23657} - 74 | {32} | {AAAAAAAAAAAAAAAA1729,AAAAAAAAAAAAA22860,AAAAAA99807,AAAAA17383,AAAAAAAAAAAAAAA67062,AAAAAAAAAAA15165,AAAAAAAAAAA50956} - 77 | {97,15,32,17,55,59,18,37,50,39} | {AAAAAAAAAAAA67946,AAAAAA54032,AAAAAAAA81587,55847,AAAAAAAAAAAAAA28620,AAAAAAAAAAAAAAAAA43052,AAAAAA75463,AAAA49534,AAAAAAAA44066} - 89 | {40,32,17,6,30,88} | {AA44673,AAAAAAAAAAA6119,AAAAAAAAAAAAAAAA23657,AAAAAAAAAAAAAAAAAA47955,AAAAAAAAAAAAAAAA33598,AAAAAAAAAAA33576,AA44673} - 98 | {38,34,32,89} | {AAAAAAAAAAAAAAAAAA71621,AAAA8857,AAAAAAAAAAAAAAAAAAA65037,AAAAAAAAAAAAAAAA31334,AAAAAAAAAA48845} - 100 | {85,32,57,39,49,84,32,3,30} | {AAAAAAA80240,AAAAAAAAAAAAAAAA1729,AAAAA60038,AAAAAAAAAAA92631,AAAAAAAA9523} -(6 rows) - -SELECT * FROM array_index_op_test WHERE i @> '{17}' ORDER BY seqno; - seqno | i | t --------+---------------------------------+------------------------------------------------------------------------------------------------------------------------------------ - 6 | {39,35,5,94,17,92,60,32} | {AAAAAAAAAAAAAAA35875,AAAAAAAAAAAAAAAA23657} - 12 | {17,99,18,52,91,72,0,43,96,23} | {AAAAA33250,AAAAAAAAAAAAAAAAAAA85420,AAAAAAAAAAA33576} - 15 | {17,14,16,63,67} | {AA6416,AAAAAAAAAA646,AAAAA95309} - 19 | {52,82,17,74,23,46,69,51,75} | {AAAAAAAAAAAAA73084,AAAAA75968,AAAAAAAAAAAAAAAA14047,AAAAAAA80240,AAAAAAAAAAAAAAAAAAA1205,A68938} - 53 | {38,17} | {AAAAAAAAAAA21658} - 65 | {61,5,76,59,17} | {AAAAAA99807,AAAAA64741,AAAAAAAAAAA53908,AA21643,AAAAAAAAA10012} - 77 | {97,15,32,17,55,59,18,37,50,39} | {AAAAAAAAAAAA67946,AAAAAA54032,AAAAAAAA81587,55847,AAAAAAAAAAAAAA28620,AAAAAAAAAAAAAAAAA43052,AAAAAA75463,AAAA49534,AAAAAAAA44066} - 89 | {40,32,17,6,30,88} | {AA44673,AAAAAAAAAAA6119,AAAAAAAAAAAAAAAA23657,AAAAAAAAAAAAAAAAAA47955,AAAAAAAAAAAAAAAA33598,AAAAAAAAAAA33576,AA44673} -(8 rows) - -SELECT * FROM array_index_op_test WHERE i && '{17}' ORDER BY seqno; - seqno | i | t --------+---------------------------------+------------------------------------------------------------------------------------------------------------------------------------ - 6 | {39,35,5,94,17,92,60,32} | {AAAAAAAAAAAAAAA35875,AAAAAAAAAAAAAAAA23657} - 12 | {17,99,18,52,91,72,0,43,96,23} | {AAAAA33250,AAAAAAAAAAAAAAAAAAA85420,AAAAAAAAAAA33576} - 15 | {17,14,16,63,67} | {AA6416,AAAAAAAAAA646,AAAAA95309} - 19 | {52,82,17,74,23,46,69,51,75} | {AAAAAAAAAAAAA73084,AAAAA75968,AAAAAAAAAAAAAAAA14047,AAAAAAA80240,AAAAAAAAAAAAAAAAAAA1205,A68938} - 53 | {38,17} | {AAAAAAAAAAA21658} - 65 | {61,5,76,59,17} | {AAAAAA99807,AAAAA64741,AAAAAAAAAAA53908,AA21643,AAAAAAAAA10012} - 77 | {97,15,32,17,55,59,18,37,50,39} | {AAAAAAAAAAAA67946,AAAAAA54032,AAAAAAAA81587,55847,AAAAAAAAAAAAAA28620,AAAAAAAAAAAAAAAAA43052,AAAAAA75463,AAAA49534,AAAAAAAA44066} - 89 | {40,32,17,6,30,88} | {AA44673,AAAAAAAAAAA6119,AAAAAAAAAAAAAAAA23657,AAAAAAAAAAAAAAAAAA47955,AAAAAAAAAAAAAAAA33598,AAAAAAAAAAA33576,AA44673} -(8 rows) - -SELECT * FROM array_index_op_test WHERE i @> '{32,17}' ORDER BY seqno; - seqno | i | t --------+---------------------------------+------------------------------------------------------------------------------------------------------------------------------------ - 6 | {39,35,5,94,17,92,60,32} | {AAAAAAAAAAAAAAA35875,AAAAAAAAAAAAAAAA23657} - 77 | {97,15,32,17,55,59,18,37,50,39} | {AAAAAAAAAAAA67946,AAAAAA54032,AAAAAAAA81587,55847,AAAAAAAAAAAAAA28620,AAAAAAAAAAAAAAAAA43052,AAAAAA75463,AAAA49534,AAAAAAAA44066} - 89 | {40,32,17,6,30,88} | {AA44673,AAAAAAAAAAA6119,AAAAAAAAAAAAAAAA23657,AAAAAAAAAAAAAAAAAA47955,AAAAAAAAAAAAAAAA33598,AAAAAAAAAAA33576,AA44673} -(3 rows) - -SELECT * FROM array_index_op_test WHERE i && '{32,17}' ORDER BY seqno; - seqno | i | t --------+---------------------------------+------------------------------------------------------------------------------------------------------------------------------------ - 6 | {39,35,5,94,17,92,60,32} | {AAAAAAAAAAAAAAA35875,AAAAAAAAAAAAAAAA23657} - 12 | {17,99,18,52,91,72,0,43,96,23} | {AAAAA33250,AAAAAAAAAAAAAAAAAAA85420,AAAAAAAAAAA33576} - 15 | {17,14,16,63,67} | {AA6416,AAAAAAAAAA646,AAAAA95309} - 19 | {52,82,17,74,23,46,69,51,75} | {AAAAAAAAAAAAA73084,AAAAA75968,AAAAAAAAAAAAAAAA14047,AAAAAAA80240,AAAAAAAAAAAAAAAAAAA1205,A68938} - 53 | {38,17} | {AAAAAAAAAAA21658} - 65 | {61,5,76,59,17} | {AAAAAA99807,AAAAA64741,AAAAAAAAAAA53908,AA21643,AAAAAAAAA10012} - 74 | {32} | {AAAAAAAAAAAAAAAA1729,AAAAAAAAAAAAA22860,AAAAAA99807,AAAAA17383,AAAAAAAAAAAAAAA67062,AAAAAAAAAAA15165,AAAAAAAAAAA50956} - 77 | {97,15,32,17,55,59,18,37,50,39} | {AAAAAAAAAAAA67946,AAAAAA54032,AAAAAAAA81587,55847,AAAAAAAAAAAAAA28620,AAAAAAAAAAAAAAAAA43052,AAAAAA75463,AAAA49534,AAAAAAAA44066} - 89 | {40,32,17,6,30,88} | {AA44673,AAAAAAAAAAA6119,AAAAAAAAAAAAAAAA23657,AAAAAAAAAAAAAAAAAA47955,AAAAAAAAAAAAAAAA33598,AAAAAAAAAAA33576,AA44673} - 98 | {38,34,32,89} | {AAAAAAAAAAAAAAAAAA71621,AAAA8857,AAAAAAAAAAAAAAAAAAA65037,AAAAAAAAAAAAAAAA31334,AAAAAAAAAA48845} - 100 | {85,32,57,39,49,84,32,3,30} | {AAAAAAA80240,AAAAAAAAAAAAAAAA1729,AAAAA60038,AAAAAAAAAAA92631,AAAAAAAA9523} -(11 rows) - -SELECT * FROM array_index_op_test WHERE i <@ '{38,34,32,89}' ORDER BY seqno; - seqno | i | t --------+---------------+---------------------------------------------------------------------------------------------------------------------------- - 40 | {34} | {AAAAAAAAAAAAAA10611,AAAAAAAAAAAAAAAAAAA1205,AAAAAAAAAAA50956,AAAAAAAAAAAAAAAA31334,AAAAA70466,AAAAAAAA81587,AAAAAAA74623} - 74 | {32} | {AAAAAAAAAAAAAAAA1729,AAAAAAAAAAAAA22860,AAAAAA99807,AAAAA17383,AAAAAAAAAAAAAAA67062,AAAAAAAAAAA15165,AAAAAAAAAAA50956} - 98 | {38,34,32,89} | {AAAAAAAAAAAAAAAAAA71621,AAAA8857,AAAAAAAAAAAAAAAAAAA65037,AAAAAAAAAAAAAAAA31334,AAAAAAAAAA48845} - 101 | {} | {} -(4 rows) - -SELECT * FROM array_index_op_test WHERE i = '{47,77}' ORDER BY seqno; - seqno | i | t --------+---------+----------------------------------------------------------------------------------------------------------------- - 95 | {47,77} | {AAAAAAAAAAAAAAAAA764,AAAAAAAAAAA74076,AAAAAAAAAA18107,AAAAA40681,AAAAAAAAAAAAAAA35875,AAAAA60038,AAAAAAA56483} -(1 row) - -SELECT * FROM array_index_op_test WHERE i = '{}' ORDER BY seqno; - seqno | i | t --------+----+---- - 101 | {} | {} -(1 row) - -SELECT * FROM array_index_op_test WHERE i @> '{}' ORDER BY seqno; - seqno | i | t --------+---------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - 1 | {92,75,71,52,64,83} | {AAAAAAAA44066,AAAAAA1059,AAAAAAAAAAA176,AAAAAAA48038} - 2 | {3,6} | {AAAAAA98232,AAAAAAAA79710,AAAAAAAAAAAAAAAAA69675,AAAAAAAAAAAAAAAA55798,AAAAAAAAA12793} - 3 | {37,64,95,43,3,41,13,30,11,43} | {AAAAAAAAAA48845,AAAAA75968,AAAAA95309,AAA54451,AAAAAAAAAA22292,AAAAAAA99836,A96617,AA17009,AAAAAAAAAAAAAA95246} - 4 | {71,39,99,55,33,75,45} | {AAAAAAAAA53663,AAAAAAAAAAAAAAA67062,AAAAAAAAAA64777,AAA99043,AAAAAAAAAAAAAAAAAAA91804,39557} - 5 | {50,42,77,50,4} | {AAAAAAAAAAAAAAAAA26540,AAAAAAA79710,AAAAAAAAAAAAAAAAAAA1205,AAAAAAAAAAA176,AAAAA95309,AAAAAAAAAAA46154,AAAAAA66777,AAAAAAAAA27249,AAAAAAAAAA64777,AAAAAAAAAAAAAAAAAAA70104} - 6 | {39,35,5,94,17,92,60,32} | {AAAAAAAAAAAAAAA35875,AAAAAAAAAAAAAAAA23657} - 7 | {12,51,88,64,8} | {AAAAAAAAAAAAAAAAAA12591,AAAAAAAAAAAAAAAAA50407,AAAAAAAAAAAA67946} - 8 | {60,84} | {AAAAAAA81898,AAAAAA1059,AAAAAAAAAAAA81511,AAAAA961,AAAAAAAAAAAAAAAA31334,AAAAA64741,AA6416,AAAAAAAAAAAAAAAAAA32918,AAAAAAAAAAAAAAAAA50407} - 9 | {56,52,35,27,80,44,81,22} | {AAAAAAAAAAAAAAA73034,AAAAAAAAAAAAA7929,AAAAAAA66161,AA88409,39557,A27153,AAAAAAAA9523,AAAAAAAAAAA99000} - 10 | {71,5,45} | {AAAAAAAAAAA21658,AAAAAAAAAAAA21089,AAA54451,AAAAAAAAAAAAAAAAAA54141,AAAAAAAAAAAAAA28620,AAAAAAAAAAA21658,AAAAAAAAAAA74076,AAAAAAAAA27249} - 11 | {41,86,74,48,22,74,47,50} | {AAAAAAAA9523,AAAAAAAAAAAA37562,AAAAAAAAAAAAAAAA14047,AAAAAAAAAAA46154,AAAA41702,AAAAAAAAAAAAAAAAA764,AAAAA62737,39557} - 12 | {17,99,18,52,91,72,0,43,96,23} | {AAAAA33250,AAAAAAAAAAAAAAAAAAA85420,AAAAAAAAAAA33576} - 13 | {3,52,34,23} | {AAAAAA98232,AAAA49534,AAAAAAAAAAA21658} - 14 | {78,57,19} | {AAAA8857,AAAAAAAAAAAAAAA73034,AAAAAAAA81587,AAAAAAAAAAAAAAA68526,AAAAA75968,AAAAAAAAAAAAAA65909,AAAAAAAAA10012,AAAAAAAAAAAAAA65909} - 15 | {17,14,16,63,67} | {AA6416,AAAAAAAAAA646,AAAAA95309} - 16 | {14,63,85,11} | {AAAAAA66777} - 17 | {7,10,81,85} | {AAAAAA43678,AAAAAAA12144,AAAAAAAAAAA50956,AAAAAAAAAAAAAAAAAAA15356} - 18 | {1} | {AAAAAAAAAAA33576,AAAAA95309,64261,AAA59323,AAAAAAAAAAAAAA95246,55847,AAAAAAAAAAAA67946,AAAAAAAAAAAAAAAAAA64374} - 19 | {52,82,17,74,23,46,69,51,75} | {AAAAAAAAAAAAA73084,AAAAA75968,AAAAAAAAAAAAAAAA14047,AAAAAAA80240,AAAAAAAAAAAAAAAAAAA1205,A68938} - 20 | {72,89,70,51,54,37,8,49,79} | {AAAAAA58494} - 21 | {2,8,65,10,5,79,43} | {AAAAAAAAAAAAAAAAA88852,AAAAAAAAAAAAAAAAAAA91804,AAAAA64669,AAAAAAAAAAAAAAAA1443,AAAAAAAAAAAAAAAA23657,AAAAA12179,AAAAAAAAAAAAAAAAA88852,AAAAAAAAAAAAAAAA31334,AAAAAAAAAAAAAAAA41303,AAAAAAAAAAAAAAAAAAA85420} - 22 | {11,6,56,62,53,30} | {AAAAAAAA72908} - 23 | {40,90,5,38,72,40,30,10,43,55} | {A6053,AAAAAAAAAAA6119,AA44673,AAAAAAAAAAAAAAAAA764,AA17009,AAAAA17383,AAAAA70514,AAAAA33250,AAAAA95309,AAAAAAAAAAAA37562} - 24 | {94,61,99,35,48} | {AAAAAAAAAAA50956,AAAAAAAAAAA15165,AAAA85070,AAAAAAAAAAAAAAA36627,AAAAA961,AAAAAAAAAA55219} - 25 | {31,1,10,11,27,79,38} | {AAAAAAAAAAAAAAAAAA59334,45449} - 26 | {71,10,9,69,75} | {47735,AAAAAAA21462,AAAAAAAAAAAAAAAAA6897,AAAAAAAAAAAAAAAAAAA91804,AAAAAAAAA72121,AAAAAAAAAAAAAAAAAAA1205,AAAAA41597,AAAA8857,AAAAAAAAAAAAAAAAAAA15356,AA17009} - 27 | {94} | {AA6416,A6053,AAAAAAA21462,AAAAAAA57334,AAAAAAAAAAAAAAAAAA12591,AA88409,AAAAAAAAAAAAA70254} - 28 | {14,33,6,34,14} | {AAAAAAAAAAAAAAA13198,AAAAAAAA69452,AAAAAAAAAAA82945,AAAAAAA12144,AAAAAAAAA72121,AAAAAAAAAA18601} - 29 | {39,21} | {AAAAAAAAAAAAAAAAA6897,AAAAAAAAAAAAAAAAAAA38885,AAAA85070,AAAAAAAAAAAAAAAAAAA70104,AAAAA66674,AAAAAAAAAAAAA62007,AAAAAAAA69452,AAAAAAA1242,AAAAAAAAAAAAAAAA1729,AAAA35194} - 30 | {26,81,47,91,34} | {AAAAAAAAAAAAAAAAAAA70104,AAAAAAA80240} - 31 | {80,24,18,21,54} | {AAAAAAAAAAAAAAA13198,AAAAAAAAAAAAAAAAAAA70415,A27153,AAAAAAAAA53663,AAAAAAAAAAAAAAAAA50407,A68938} - 32 | {58,79,82,80,67,75,98,10,41} | {AAAAAAAAAAAAAAAAAA61286,AAA54451,AAAAAAAAAAAAAAAAAAA87527,A96617,51533} - 33 | {74,73} | {A85417,AAAAAAA56483,AAAAA17383,AAAAAAAAAAAAA62159,AAAAAAAAAAAA52814,AAAAAAAAAAAAA85723,AAAAAAAAAAAAAAAAAA55796} - 34 | {70,45} | {AAAAAAAAAAAAAAAAAA71621,AAAAAAAAAAAAAA28620,AAAAAAAAAA55219,AAAAAAAA23648,AAAAAAAAAA22292,AAAAAAA1242} - 35 | {23,40} | {AAAAAAAAAAAA52814,AAAA48949,AAAAAAAAA34727,AAAA8857,AAAAAAAAAAAAAAAAAAA62179,AAAAAAAAAAAAAAA68526,AAAAAAA99836,AAAAAAAA50094,AAAA91194,AAAAAAAAAAAAA73084} - 36 | {79,82,14,52,30,5,79} | {AAAAAAAAA53663,AAAAAAAAAAAAAAAA55798,AAAAAAAAAAAAAAAAAAA89194,AA88409,AAAAAAAAAAAAAAA81326,AAAAAAAAAAAAAAAAA63050,AAAAAAAAAAAAAAAA33598} - 37 | {53,11,81,39,3,78,58,64,74} | {AAAAAAAAAAAAAAAAAAA17075,AAAAAAA66161,AAAAAAAA23648,AAAAAAAAAAAAAA10611} - 38 | {59,5,4,95,28} | {AAAAAAAAAAA82945,A96617,47735,AAAAA12179,AAAAA64669,AAAAAA99807,AA74433,AAAAAAAAAAAAAAAAA59387} - 39 | {82,43,99,16,74} | {AAAAAAAAAAAAAAA67062,AAAAAAA57334,AAAAAAAAAAAAAA65909,A27153,AAAAAAAAAAAAAAAAAAA17075,AAAAAAAAAAAAAAAAA43052,AAAAAAAAAA64777,AAAAAAAAAAAA81511,AAAAAAAAAAAAAA65909,AAAAAAAAAAAAAA28620} - 40 | {34} | {AAAAAAAAAAAAAA10611,AAAAAAAAAAAAAAAAAAA1205,AAAAAAAAAAA50956,AAAAAAAAAAAAAAAA31334,AAAAA70466,AAAAAAAA81587,AAAAAAA74623} - 41 | {19,26,63,12,93,73,27,94} | {AAAAAAA79710,AAAAAAAAAA55219,AAAA41702,AAAAAAAAAAAAAAAAAAA17075,AAAAAAAAAAAAAAAAAA71621,AAAAAAAAAAAAAAAAA63050,AAAAAAA99836,AAAAAAAAAAAAAA8666} - 42 | {15,76,82,75,8,91} | {AAAAAAAAAAA176,AAAAAA38063,45449,AAAAAA54032,AAAAAAA81898,AA6416,AAAAAAAAAAAAAAAAAAA62179,45449,AAAAA60038,AAAAAAAA81587} - 43 | {39,87,91,97,79,28} | {AAAAAAAAAAA74076,A96617,AAAAAAAAAAAAAAAAAAA89194,AAAAAAAAAAAAAAAAAA55796,AAAAAAAAAAAAAAAA23657,AAAAAAAAAAAA67946} - 44 | {40,58,68,29,54} | {AAAAAAA81898,AAAAAA66777,AAAAAA98232} - 45 | {99,45} | {AAAAAAAA72908,AAAAAAAAAAAAAAAAAAA17075,AA88409,AAAAAAAAAAAAAAAAAA36842,AAAAAAA48038,AAAAAAAAAAAAAA10611} - 46 | {53,24} | {AAAAAAAAAAA53908,AAAAAA54032,AAAAA17383,AAAA48949,AAAAAAAAAA18601,AAAAA64669,45449,AAAAAAAAAAA98051,AAAAAAAAAAAAAAAAAA71621} - 47 | {98,23,64,12,75,61} | {AAA59323,AAAAA95309,AAAAAAAAAAAAAAAA31334,AAAAAAAAA27249,AAAAA17383,AAAAAAAAAAAA37562,AAAAAA1059,A84822,55847,AAAAA70466} - 48 | {76,14} | {AAAAAAAAAAAAA59671,AAAAAAAAAAAAAAAAAAA91804,AAAAAA66777,AAAAAAAAAAAAAAAAAAA89194,AAAAAAAAAAAAAAA36627,AAAAAAAAAAAAAAAAAAA17075,AAAAAAAAAAAAA73084,AAAAAAA79710,AAAAAAAAAAAAAAA40402,AAAAAAAAAAAAAAAAAAA65037} - 49 | {56,5,54,37,49} | {AA21643,AAAAAAAAAAA92631,AAAAAAAA81587} - 50 | {20,12,37,64,93} | {AAAAAAAAAA5483,AAAAAAAAAAAAAAAAAAA1205,AA6416,AAAAAAAAAAAAAAAAA63050,AAAAAAAAAAAAAAAAAA47955} - 51 | {47} | {AAAAAAAAAAAAAA96505,AAAAAAAAAAAAAAAAAA36842,AAAAA95309,AAAAAAAA81587,AA6416,AAAA91194,AAAAAA58494,AAAAAA1059,AAAAAAAA69452} - 52 | {89,0} | {AAAAAAAAAAAAAAAAAA47955,AAAAAAA48038,AAAAAAAAAAAAAAAAA43052,AAAAAAAAAAAAA73084,AAAAA70466,AAAAAAAAAAAAAAAAA764,AAAAAAAAAAA46154,AA66862} - 53 | {38,17} | {AAAAAAAAAAA21658} - 54 | {70,47} | {AAAAAAAAAAAAAAAAAA54141,AAAAA40681,AAAAAAA48038,AAAAAAAAAAAAAAAA29150,AAAAA41597,AAAAAAAAAAAAAAAAAA59334,AA15322} - 55 | {47,79,47,64,72,25,71,24,93} | {AAAAAAAAAAAAAAAAAA55796,AAAAA62737} - 56 | {33,7,60,54,93,90,77,85,39} | {AAAAAAAAAAAAAAAAAA32918,AA42406} - 57 | {23,45,10,42,36,21,9,96} | {AAAAAAAAAAAAAAAAAAA70415} - 58 | {92} | {AAAAAAAAAAAAAAAA98414,AAAAAAAA23648,AAAAAAAAAAAAAAAAAA55796,AA25381,AAAAAAAAAAA6119} - 59 | {9,69,46,77} | {39557,AAAAAAA89932,AAAAAAAAAAAAAAAAA43052,AAAAAAAAAAAAAAAAA26540,AAA20874,AA6416,AAAAAAAAAAAAAAAAAA47955} - 60 | {62,2,59,38,89} | {AAAAAAA89932,AAAAAAAAAAAAAAAAAAA15356,AA99927,AA17009,AAAAAAAAAAAAAAA35875} - 61 | {72,2,44,95,54,54,13} | {AAAAAAAAAAAAAAAAAAA91804} - 62 | {83,72,29,73} | {AAAAAAAAAAAAA15097,AAAA8857,AAAAAAAAAAAA35809,AAAAAAAAAAAA52814,AAAAAAAAAAAAAAAAAAA38885,AAAAAAAAAAAAAAAAAA24183,AAAAAA43678,A96617} - 63 | {11,4,61,87} | {AAAAAAAAA27249,AAAAAAAAAAAAAAAAAA32918,AAAAAAAAAAAAAAA13198,AAA20874,39557,51533,AAAAAAAAAAA53908,AAAAAAAAAAAAAA96505,AAAAAAAA78938} - 64 | {26,19,34,24,81,78} | {A96617,AAAAAAAAAAAAAAAAAAA70104,A68938,AAAAAAAAAAA53908,AAAAAAAAAAAAAAA453,AA17009,AAAAAAA80240} - 65 | {61,5,76,59,17} | {AAAAAA99807,AAAAA64741,AAAAAAAAAAA53908,AA21643,AAAAAAAAA10012} - 66 | {31,23,70,52,4,33,48,25} | {AAAAAAAAAAAAAAAAA69675,AAAAAAAA50094,AAAAAAAAAAA92631,AAAA35194,39557,AAAAAAA99836} - 67 | {31,94,7,10} | {AAAAAA38063,A96617,AAAA35194,AAAAAAAAAAAA67946} - 68 | {90,43,38} | {AA75092,AAAAAAAAAAAAAAAAA69675,AAAAAAAAAAA92631,AAAAAAAAA10012,AAAAAAAAAAAAA7929,AA21643} - 69 | {67,35,99,85,72,86,44} | {AAAAAAAAAAAAAAAAAAA1205,AAAAAAAA50094,AAAAAAAAAAAAAAAA1729,AAAAAAAAAAAAAAAAAA47955} - 70 | {56,70,83} | {AAAA41702,AAAAAAAAAAA82945,AA21643,AAAAAAAAAAA99000,A27153,AA25381,AAAAAAAAAAAAAA96505,AAAAAAA1242} - 71 | {74,26} | {AAAAAAAAAAA50956,AA74433,AAAAAAA21462,AAAAAAAAAAAAAAAAAAA17075,AAAAAAAAAAAAAAA36627,AAAAAAAAAAAAA70254,AAAAAAAAAA43419,39557} - 72 | {22,1,16,78,20,91,83} | {47735,AAAAAAA56483,AAAAAAAAAAAAA93788,AA42406,AAAAAAAAAAAAA73084,AAAAAAAA72908,AAAAAAAAAAAAAAAAAA61286,AAAAA66674,AAAAAAAAAAAAAAAAA50407} - 73 | {88,25,96,78,65,15,29,19} | {AAA54451,AAAAAAAAA27249,AAAAAAA9228,AAAAAAAAAAAAAAA67062,AAAAAAAAAAAAAAAAAAA70415,AAAAA17383,AAAAAAAAAAAAAAAA33598} - 74 | {32} | {AAAAAAAAAAAAAAAA1729,AAAAAAAAAAAAA22860,AAAAAA99807,AAAAA17383,AAAAAAAAAAAAAAA67062,AAAAAAAAAAA15165,AAAAAAAAAAA50956} - 75 | {12,96,83,24,71,89,55} | {AAAA48949,AAAAAAAA29716,AAAAAAAAAAAAAAAAAAA1205,AAAAAAAAAAAA67946,AAAAAAAAAAAAAAAA29150,AAA28075,AAAAAAAAAAAAAAAAA43052} - 76 | {92,55,10,7} | {AAAAAAAAAAAAAAA67062} - 77 | {97,15,32,17,55,59,18,37,50,39} | {AAAAAAAAAAAA67946,AAAAAA54032,AAAAAAAA81587,55847,AAAAAAAAAAAAAA28620,AAAAAAAAAAAAAAAAA43052,AAAAAA75463,AAAA49534,AAAAAAAA44066} - 78 | {55,89,44,84,34} | {AAAAAAAAAAA6119,AAAAAAAAAAAAAA8666,AA99927,AA42406,AAAAAAA81898,AAAAAAA9228,AAAAAAAAAAA92631,AA21643,AAAAAAAAAAAAAA28620} - 79 | {45} | {AAAAAAAAAA646,AAAAAAAAAAAAAAAAAAA70415,AAAAAA43678,AAAAAAAA72908} - 80 | {74,89,44,80,0} | {AAAA35194,AAAAAAAA79710,AAA20874,AAAAAAAAAAAAAAAAAAA70104,AAAAAAAAAAAAA73084,AAAAAAA57334,AAAAAAA9228,AAAAAAAAAAAAA62007} - 81 | {63,77,54,48,61,53,97} | {AAAAAAAAAAAAAAA81326,AAAAAAAAAA22292,AA25381,AAAAAAAAAAA74076,AAAAAAA81898,AAAAAAAAA72121} - 82 | {34,60,4,79,78,16,86,89,42,50} | {AAAAA40681,AAAAAAAAAAAAAAAAAA12591,AAAAAAA80240,AAAAAAAAAAAAAAAA55798,AAAAAAAAAAAAAAAAAAA70104} - 83 | {14,10} | {AAAAAAAAAA22292,AAAAAAAAAAAAA70254,AAAAAAAAAAA6119} - 84 | {11,83,35,13,96,94} | {AAAAA95309,AAAAAAAAAAAAAAAAAA32918,AAAAAAAAAAAAAAAAAA24183} - 85 | {39,60} | {AAAAAAAAAAAAAAAA55798,AAAAAAAAAA22292,AAAAAAA66161,AAAAAAA21462,AAAAAAAAAAAAAAAAAA12591,55847,AAAAAA98232,AAAAAAAAAAA46154} - 86 | {33,81,72,74,45,36,82} | {AAAAAAAA81587,AAAAAAAAAAAAAA96505,45449,AAAA80176} - 87 | {57,27,50,12,97,68} | {AAAAAAAAAAAAAAAAA26540,AAAAAAAAA10012,AAAAAAAAAAAA35809,AAAAAAAAAAAAAAAA29150,AAAAAAAAAAA82945,AAAAAA66777,31228,AAAAAAAAAAAAAAAA23657,AAAAAAAAAAAAAA28620,AAAAAAAAAAAAAA96505} - 88 | {41,90,77,24,6,24} | {AAAA35194,AAAA35194,AAAAAAA80240,AAAAAAAAAAA46154,AAAAAA58494,AAAAAAAAAAAAAAAAAAA17075,AAAAAAAAAAAAAAAAAA59334,AAAAAAAAAAAAAAAAAAA91804,AA74433} - 89 | {40,32,17,6,30,88} | {AA44673,AAAAAAAAAAA6119,AAAAAAAAAAAAAAAA23657,AAAAAAAAAAAAAAAAAA47955,AAAAAAAAAAAAAAAA33598,AAAAAAAAAAA33576,AA44673} - 90 | {88,75} | {AAAAA60038,AAAAAAAA23648,AAAAAAAAAAA99000,AAAA41702,AAAAAAAAAAAAA22860,AAAAAAAAAAAAAAA68526} - 91 | {78} | {AAAAAAAAAAAAA62007,AAA99043} - 92 | {85,63,49,45} | {AAAAAAA89932,AAAAAAAAAAAAA22860,AAAAAAAAAAAAAAAAAAA1205,AAAAAAAAAAAA21089} - 93 | {11} | {AAAAAAAAAAA176,AAAAAAAAAAAAAA8666,AAAAAAAAAAAAAAA453,AAAAAAAAAAAAA85723,A68938,AAAAAAAAAAAAA9821,AAAAAAA48038,AAAAAAAAAAAAAAAAA59387,AA99927,AAAAA17383} - 94 | {98,9,85,62,88,91,60,61,38,86} | {AAAAAAAA81587,AAAAA17383,AAAAAAAA81587} - 95 | {47,77} | {AAAAAAAAAAAAAAAAA764,AAAAAAAAAAA74076,AAAAAAAAAA18107,AAAAA40681,AAAAAAAAAAAAAAA35875,AAAAA60038,AAAAAAA56483} - 96 | {23,97,43} | {AAAAAAAAAA646,A87088} - 97 | {54,2,86,65} | {47735,AAAAAAA99836,AAAAAAAAAAAAAAAAA6897,AAAAAAAAAAAAAAAA29150,AAAAAAA80240,AAAAAAAAAAAAAAAA98414,AAAAAAA56483,AAAAAAAAAAAAAAAA29150,AAAAAAA39692,AA21643} - 98 | {38,34,32,89} | {AAAAAAAAAAAAAAAAAA71621,AAAA8857,AAAAAAAAAAAAAAAAAAA65037,AAAAAAAAAAAAAAAA31334,AAAAAAAAAA48845} - 99 | {37,86} | {AAAAAAAAAAAAAAAAAA32918,AAAAA70514,AAAAAAAAA10012,AAAAAAAAAAAAAAAAA59387,AAAAAAAAAA64777,AAAAAAAAAAAAAAAAAAA15356} - 100 | {85,32,57,39,49,84,32,3,30} | {AAAAAAA80240,AAAAAAAAAAAAAAAA1729,AAAAA60038,AAAAAAAAAAA92631,AAAAAAAA9523} - 101 | {} | {} - 102 | {NULL} | {NULL} -(102 rows) - -SELECT * FROM array_index_op_test WHERE i && '{}' ORDER BY seqno; - seqno | i | t --------+---+--- -(0 rows) - -SELECT * FROM array_index_op_test WHERE i <@ '{}' ORDER BY seqno; - seqno | i | t --------+----+---- - 101 | {} | {} -(1 row) - -CREATE INDEX textarrayidx ON array_index_op_test USING gin (t); -explain (costs off) -SELECT * FROM array_index_op_test WHERE t @> '{AAAAAAAA72908}' ORDER BY seqno; - QUERY PLAN ------------------------------------------------------------- - Sort - Sort Key: seqno - -> Bitmap Heap Scan on array_index_op_test - Recheck Cond: (t @> '{AAAAAAAA72908}'::text[]) - -> Bitmap Index Scan on textarrayidx - Index Cond: (t @> '{AAAAAAAA72908}'::text[]) -(6 rows) - -SELECT * FROM array_index_op_test WHERE t @> '{AAAAAAAA72908}' ORDER BY seqno; - seqno | i | t --------+-----------------------+-------------------------------------------------------------------------------------------------------------------------------------------- - 22 | {11,6,56,62,53,30} | {AAAAAAAA72908} - 45 | {99,45} | {AAAAAAAA72908,AAAAAAAAAAAAAAAAAAA17075,AA88409,AAAAAAAAAAAAAAAAAA36842,AAAAAAA48038,AAAAAAAAAAAAAA10611} - 72 | {22,1,16,78,20,91,83} | {47735,AAAAAAA56483,AAAAAAAAAAAAA93788,AA42406,AAAAAAAAAAAAA73084,AAAAAAAA72908,AAAAAAAAAAAAAAAAAA61286,AAAAA66674,AAAAAAAAAAAAAAAAA50407} - 79 | {45} | {AAAAAAAAAA646,AAAAAAAAAAAAAAAAAAA70415,AAAAAA43678,AAAAAAAA72908} -(4 rows) - -SELECT * FROM array_index_op_test WHERE t && '{AAAAAAAA72908}' ORDER BY seqno; - seqno | i | t --------+-----------------------+-------------------------------------------------------------------------------------------------------------------------------------------- - 22 | {11,6,56,62,53,30} | {AAAAAAAA72908} - 45 | {99,45} | {AAAAAAAA72908,AAAAAAAAAAAAAAAAAAA17075,AA88409,AAAAAAAAAAAAAAAAAA36842,AAAAAAA48038,AAAAAAAAAAAAAA10611} - 72 | {22,1,16,78,20,91,83} | {47735,AAAAAAA56483,AAAAAAAAAAAAA93788,AA42406,AAAAAAAAAAAAA73084,AAAAAAAA72908,AAAAAAAAAAAAAAAAAA61286,AAAAA66674,AAAAAAAAAAAAAAAAA50407} - 79 | {45} | {AAAAAAAAAA646,AAAAAAAAAAAAAAAAAAA70415,AAAAAA43678,AAAAAAAA72908} -(4 rows) - -SELECT * FROM array_index_op_test WHERE t @> '{AAAAAAAAAA646}' ORDER BY seqno; - seqno | i | t --------+------------------+-------------------------------------------------------------------- - 15 | {17,14,16,63,67} | {AA6416,AAAAAAAAAA646,AAAAA95309} - 79 | {45} | {AAAAAAAAAA646,AAAAAAAAAAAAAAAAAAA70415,AAAAAA43678,AAAAAAAA72908} - 96 | {23,97,43} | {AAAAAAAAAA646,A87088} -(3 rows) - -SELECT * FROM array_index_op_test WHERE t && '{AAAAAAAAAA646}' ORDER BY seqno; - seqno | i | t --------+------------------+-------------------------------------------------------------------- - 15 | {17,14,16,63,67} | {AA6416,AAAAAAAAAA646,AAAAA95309} - 79 | {45} | {AAAAAAAAAA646,AAAAAAAAAAAAAAAAAAA70415,AAAAAA43678,AAAAAAAA72908} - 96 | {23,97,43} | {AAAAAAAAAA646,A87088} -(3 rows) - -SELECT * FROM array_index_op_test WHERE t @> '{AAAAAAAA72908,AAAAAAAAAA646}' ORDER BY seqno; - seqno | i | t --------+------+-------------------------------------------------------------------- - 79 | {45} | {AAAAAAAAAA646,AAAAAAAAAAAAAAAAAAA70415,AAAAAA43678,AAAAAAAA72908} -(1 row) - -SELECT * FROM array_index_op_test WHERE t && '{AAAAAAAA72908,AAAAAAAAAA646}' ORDER BY seqno; - seqno | i | t --------+-----------------------+-------------------------------------------------------------------------------------------------------------------------------------------- - 15 | {17,14,16,63,67} | {AA6416,AAAAAAAAAA646,AAAAA95309} - 22 | {11,6,56,62,53,30} | {AAAAAAAA72908} - 45 | {99,45} | {AAAAAAAA72908,AAAAAAAAAAAAAAAAAAA17075,AA88409,AAAAAAAAAAAAAAAAAA36842,AAAAAAA48038,AAAAAAAAAAAAAA10611} - 72 | {22,1,16,78,20,91,83} | {47735,AAAAAAA56483,AAAAAAAAAAAAA93788,AA42406,AAAAAAAAAAAAA73084,AAAAAAAA72908,AAAAAAAAAAAAAAAAAA61286,AAAAA66674,AAAAAAAAAAAAAAAAA50407} - 79 | {45} | {AAAAAAAAAA646,AAAAAAAAAAAAAAAAAAA70415,AAAAAA43678,AAAAAAAA72908} - 96 | {23,97,43} | {AAAAAAAAAA646,A87088} -(6 rows) - -SELECT * FROM array_index_op_test WHERE t <@ '{AAAAAAAA72908,AAAAAAAAAAAAAAAAAAA17075,AA88409,AAAAAAAAAAAAAAAAAA36842,AAAAAAA48038,AAAAAAAAAAAAAA10611}' ORDER BY seqno; - seqno | i | t --------+--------------------+----------------------------------------------------------------------------------------------------------- - 22 | {11,6,56,62,53,30} | {AAAAAAAA72908} - 45 | {99,45} | {AAAAAAAA72908,AAAAAAAAAAAAAAAAAAA17075,AA88409,AAAAAAAAAAAAAAAAAA36842,AAAAAAA48038,AAAAAAAAAAAAAA10611} - 101 | {} | {} -(3 rows) - -SELECT * FROM array_index_op_test WHERE t = '{AAAAAAAAAA646,A87088}' ORDER BY seqno; - seqno | i | t --------+------------+------------------------ - 96 | {23,97,43} | {AAAAAAAAAA646,A87088} -(1 row) - -SELECT * FROM array_index_op_test WHERE t = '{}' ORDER BY seqno; - seqno | i | t --------+----+---- - 101 | {} | {} -(1 row) - -SELECT * FROM array_index_op_test WHERE t @> '{}' ORDER BY seqno; - seqno | i | t --------+---------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - 1 | {92,75,71,52,64,83} | {AAAAAAAA44066,AAAAAA1059,AAAAAAAAAAA176,AAAAAAA48038} - 2 | {3,6} | {AAAAAA98232,AAAAAAAA79710,AAAAAAAAAAAAAAAAA69675,AAAAAAAAAAAAAAAA55798,AAAAAAAAA12793} - 3 | {37,64,95,43,3,41,13,30,11,43} | {AAAAAAAAAA48845,AAAAA75968,AAAAA95309,AAA54451,AAAAAAAAAA22292,AAAAAAA99836,A96617,AA17009,AAAAAAAAAAAAAA95246} - 4 | {71,39,99,55,33,75,45} | {AAAAAAAAA53663,AAAAAAAAAAAAAAA67062,AAAAAAAAAA64777,AAA99043,AAAAAAAAAAAAAAAAAAA91804,39557} - 5 | {50,42,77,50,4} | {AAAAAAAAAAAAAAAAA26540,AAAAAAA79710,AAAAAAAAAAAAAAAAAAA1205,AAAAAAAAAAA176,AAAAA95309,AAAAAAAAAAA46154,AAAAAA66777,AAAAAAAAA27249,AAAAAAAAAA64777,AAAAAAAAAAAAAAAAAAA70104} - 6 | {39,35,5,94,17,92,60,32} | {AAAAAAAAAAAAAAA35875,AAAAAAAAAAAAAAAA23657} - 7 | {12,51,88,64,8} | {AAAAAAAAAAAAAAAAAA12591,AAAAAAAAAAAAAAAAA50407,AAAAAAAAAAAA67946} - 8 | {60,84} | {AAAAAAA81898,AAAAAA1059,AAAAAAAAAAAA81511,AAAAA961,AAAAAAAAAAAAAAAA31334,AAAAA64741,AA6416,AAAAAAAAAAAAAAAAAA32918,AAAAAAAAAAAAAAAAA50407} - 9 | {56,52,35,27,80,44,81,22} | {AAAAAAAAAAAAAAA73034,AAAAAAAAAAAAA7929,AAAAAAA66161,AA88409,39557,A27153,AAAAAAAA9523,AAAAAAAAAAA99000} - 10 | {71,5,45} | {AAAAAAAAAAA21658,AAAAAAAAAAAA21089,AAA54451,AAAAAAAAAAAAAAAAAA54141,AAAAAAAAAAAAAA28620,AAAAAAAAAAA21658,AAAAAAAAAAA74076,AAAAAAAAA27249} - 11 | {41,86,74,48,22,74,47,50} | {AAAAAAAA9523,AAAAAAAAAAAA37562,AAAAAAAAAAAAAAAA14047,AAAAAAAAAAA46154,AAAA41702,AAAAAAAAAAAAAAAAA764,AAAAA62737,39557} - 12 | {17,99,18,52,91,72,0,43,96,23} | {AAAAA33250,AAAAAAAAAAAAAAAAAAA85420,AAAAAAAAAAA33576} - 13 | {3,52,34,23} | {AAAAAA98232,AAAA49534,AAAAAAAAAAA21658} - 14 | {78,57,19} | {AAAA8857,AAAAAAAAAAAAAAA73034,AAAAAAAA81587,AAAAAAAAAAAAAAA68526,AAAAA75968,AAAAAAAAAAAAAA65909,AAAAAAAAA10012,AAAAAAAAAAAAAA65909} - 15 | {17,14,16,63,67} | {AA6416,AAAAAAAAAA646,AAAAA95309} - 16 | {14,63,85,11} | {AAAAAA66777} - 17 | {7,10,81,85} | {AAAAAA43678,AAAAAAA12144,AAAAAAAAAAA50956,AAAAAAAAAAAAAAAAAAA15356} - 18 | {1} | {AAAAAAAAAAA33576,AAAAA95309,64261,AAA59323,AAAAAAAAAAAAAA95246,55847,AAAAAAAAAAAA67946,AAAAAAAAAAAAAAAAAA64374} - 19 | {52,82,17,74,23,46,69,51,75} | {AAAAAAAAAAAAA73084,AAAAA75968,AAAAAAAAAAAAAAAA14047,AAAAAAA80240,AAAAAAAAAAAAAAAAAAA1205,A68938} - 20 | {72,89,70,51,54,37,8,49,79} | {AAAAAA58494} - 21 | {2,8,65,10,5,79,43} | {AAAAAAAAAAAAAAAAA88852,AAAAAAAAAAAAAAAAAAA91804,AAAAA64669,AAAAAAAAAAAAAAAA1443,AAAAAAAAAAAAAAAA23657,AAAAA12179,AAAAAAAAAAAAAAAAA88852,AAAAAAAAAAAAAAAA31334,AAAAAAAAAAAAAAAA41303,AAAAAAAAAAAAAAAAAAA85420} - 22 | {11,6,56,62,53,30} | {AAAAAAAA72908} - 23 | {40,90,5,38,72,40,30,10,43,55} | {A6053,AAAAAAAAAAA6119,AA44673,AAAAAAAAAAAAAAAAA764,AA17009,AAAAA17383,AAAAA70514,AAAAA33250,AAAAA95309,AAAAAAAAAAAA37562} - 24 | {94,61,99,35,48} | {AAAAAAAAAAA50956,AAAAAAAAAAA15165,AAAA85070,AAAAAAAAAAAAAAA36627,AAAAA961,AAAAAAAAAA55219} - 25 | {31,1,10,11,27,79,38} | {AAAAAAAAAAAAAAAAAA59334,45449} - 26 | {71,10,9,69,75} | {47735,AAAAAAA21462,AAAAAAAAAAAAAAAAA6897,AAAAAAAAAAAAAAAAAAA91804,AAAAAAAAA72121,AAAAAAAAAAAAAAAAAAA1205,AAAAA41597,AAAA8857,AAAAAAAAAAAAAAAAAAA15356,AA17009} - 27 | {94} | {AA6416,A6053,AAAAAAA21462,AAAAAAA57334,AAAAAAAAAAAAAAAAAA12591,AA88409,AAAAAAAAAAAAA70254} - 28 | {14,33,6,34,14} | {AAAAAAAAAAAAAAA13198,AAAAAAAA69452,AAAAAAAAAAA82945,AAAAAAA12144,AAAAAAAAA72121,AAAAAAAAAA18601} - 29 | {39,21} | {AAAAAAAAAAAAAAAAA6897,AAAAAAAAAAAAAAAAAAA38885,AAAA85070,AAAAAAAAAAAAAAAAAAA70104,AAAAA66674,AAAAAAAAAAAAA62007,AAAAAAAA69452,AAAAAAA1242,AAAAAAAAAAAAAAAA1729,AAAA35194} - 30 | {26,81,47,91,34} | {AAAAAAAAAAAAAAAAAAA70104,AAAAAAA80240} - 31 | {80,24,18,21,54} | {AAAAAAAAAAAAAAA13198,AAAAAAAAAAAAAAAAAAA70415,A27153,AAAAAAAAA53663,AAAAAAAAAAAAAAAAA50407,A68938} - 32 | {58,79,82,80,67,75,98,10,41} | {AAAAAAAAAAAAAAAAAA61286,AAA54451,AAAAAAAAAAAAAAAAAAA87527,A96617,51533} - 33 | {74,73} | {A85417,AAAAAAA56483,AAAAA17383,AAAAAAAAAAAAA62159,AAAAAAAAAAAA52814,AAAAAAAAAAAAA85723,AAAAAAAAAAAAAAAAAA55796} - 34 | {70,45} | {AAAAAAAAAAAAAAAAAA71621,AAAAAAAAAAAAAA28620,AAAAAAAAAA55219,AAAAAAAA23648,AAAAAAAAAA22292,AAAAAAA1242} - 35 | {23,40} | {AAAAAAAAAAAA52814,AAAA48949,AAAAAAAAA34727,AAAA8857,AAAAAAAAAAAAAAAAAAA62179,AAAAAAAAAAAAAAA68526,AAAAAAA99836,AAAAAAAA50094,AAAA91194,AAAAAAAAAAAAA73084} - 36 | {79,82,14,52,30,5,79} | {AAAAAAAAA53663,AAAAAAAAAAAAAAAA55798,AAAAAAAAAAAAAAAAAAA89194,AA88409,AAAAAAAAAAAAAAA81326,AAAAAAAAAAAAAAAAA63050,AAAAAAAAAAAAAAAA33598} - 37 | {53,11,81,39,3,78,58,64,74} | {AAAAAAAAAAAAAAAAAAA17075,AAAAAAA66161,AAAAAAAA23648,AAAAAAAAAAAAAA10611} - 38 | {59,5,4,95,28} | {AAAAAAAAAAA82945,A96617,47735,AAAAA12179,AAAAA64669,AAAAAA99807,AA74433,AAAAAAAAAAAAAAAAA59387} - 39 | {82,43,99,16,74} | {AAAAAAAAAAAAAAA67062,AAAAAAA57334,AAAAAAAAAAAAAA65909,A27153,AAAAAAAAAAAAAAAAAAA17075,AAAAAAAAAAAAAAAAA43052,AAAAAAAAAA64777,AAAAAAAAAAAA81511,AAAAAAAAAAAAAA65909,AAAAAAAAAAAAAA28620} - 40 | {34} | {AAAAAAAAAAAAAA10611,AAAAAAAAAAAAAAAAAAA1205,AAAAAAAAAAA50956,AAAAAAAAAAAAAAAA31334,AAAAA70466,AAAAAAAA81587,AAAAAAA74623} - 41 | {19,26,63,12,93,73,27,94} | {AAAAAAA79710,AAAAAAAAAA55219,AAAA41702,AAAAAAAAAAAAAAAAAAA17075,AAAAAAAAAAAAAAAAAA71621,AAAAAAAAAAAAAAAAA63050,AAAAAAA99836,AAAAAAAAAAAAAA8666} - 42 | {15,76,82,75,8,91} | {AAAAAAAAAAA176,AAAAAA38063,45449,AAAAAA54032,AAAAAAA81898,AA6416,AAAAAAAAAAAAAAAAAAA62179,45449,AAAAA60038,AAAAAAAA81587} - 43 | {39,87,91,97,79,28} | {AAAAAAAAAAA74076,A96617,AAAAAAAAAAAAAAAAAAA89194,AAAAAAAAAAAAAAAAAA55796,AAAAAAAAAAAAAAAA23657,AAAAAAAAAAAA67946} - 44 | {40,58,68,29,54} | {AAAAAAA81898,AAAAAA66777,AAAAAA98232} - 45 | {99,45} | {AAAAAAAA72908,AAAAAAAAAAAAAAAAAAA17075,AA88409,AAAAAAAAAAAAAAAAAA36842,AAAAAAA48038,AAAAAAAAAAAAAA10611} - 46 | {53,24} | {AAAAAAAAAAA53908,AAAAAA54032,AAAAA17383,AAAA48949,AAAAAAAAAA18601,AAAAA64669,45449,AAAAAAAAAAA98051,AAAAAAAAAAAAAAAAAA71621} - 47 | {98,23,64,12,75,61} | {AAA59323,AAAAA95309,AAAAAAAAAAAAAAAA31334,AAAAAAAAA27249,AAAAA17383,AAAAAAAAAAAA37562,AAAAAA1059,A84822,55847,AAAAA70466} - 48 | {76,14} | {AAAAAAAAAAAAA59671,AAAAAAAAAAAAAAAAAAA91804,AAAAAA66777,AAAAAAAAAAAAAAAAAAA89194,AAAAAAAAAAAAAAA36627,AAAAAAAAAAAAAAAAAAA17075,AAAAAAAAAAAAA73084,AAAAAAA79710,AAAAAAAAAAAAAAA40402,AAAAAAAAAAAAAAAAAAA65037} - 49 | {56,5,54,37,49} | {AA21643,AAAAAAAAAAA92631,AAAAAAAA81587} - 50 | {20,12,37,64,93} | {AAAAAAAAAA5483,AAAAAAAAAAAAAAAAAAA1205,AA6416,AAAAAAAAAAAAAAAAA63050,AAAAAAAAAAAAAAAAAA47955} - 51 | {47} | {AAAAAAAAAAAAAA96505,AAAAAAAAAAAAAAAAAA36842,AAAAA95309,AAAAAAAA81587,AA6416,AAAA91194,AAAAAA58494,AAAAAA1059,AAAAAAAA69452} - 52 | {89,0} | {AAAAAAAAAAAAAAAAAA47955,AAAAAAA48038,AAAAAAAAAAAAAAAAA43052,AAAAAAAAAAAAA73084,AAAAA70466,AAAAAAAAAAAAAAAAA764,AAAAAAAAAAA46154,AA66862} - 53 | {38,17} | {AAAAAAAAAAA21658} - 54 | {70,47} | {AAAAAAAAAAAAAAAAAA54141,AAAAA40681,AAAAAAA48038,AAAAAAAAAAAAAAAA29150,AAAAA41597,AAAAAAAAAAAAAAAAAA59334,AA15322} - 55 | {47,79,47,64,72,25,71,24,93} | {AAAAAAAAAAAAAAAAAA55796,AAAAA62737} - 56 | {33,7,60,54,93,90,77,85,39} | {AAAAAAAAAAAAAAAAAA32918,AA42406} - 57 | {23,45,10,42,36,21,9,96} | {AAAAAAAAAAAAAAAAAAA70415} - 58 | {92} | {AAAAAAAAAAAAAAAA98414,AAAAAAAA23648,AAAAAAAAAAAAAAAAAA55796,AA25381,AAAAAAAAAAA6119} - 59 | {9,69,46,77} | {39557,AAAAAAA89932,AAAAAAAAAAAAAAAAA43052,AAAAAAAAAAAAAAAAA26540,AAA20874,AA6416,AAAAAAAAAAAAAAAAAA47955} - 60 | {62,2,59,38,89} | {AAAAAAA89932,AAAAAAAAAAAAAAAAAAA15356,AA99927,AA17009,AAAAAAAAAAAAAAA35875} - 61 | {72,2,44,95,54,54,13} | {AAAAAAAAAAAAAAAAAAA91804} - 62 | {83,72,29,73} | {AAAAAAAAAAAAA15097,AAAA8857,AAAAAAAAAAAA35809,AAAAAAAAAAAA52814,AAAAAAAAAAAAAAAAAAA38885,AAAAAAAAAAAAAAAAAA24183,AAAAAA43678,A96617} - 63 | {11,4,61,87} | {AAAAAAAAA27249,AAAAAAAAAAAAAAAAAA32918,AAAAAAAAAAAAAAA13198,AAA20874,39557,51533,AAAAAAAAAAA53908,AAAAAAAAAAAAAA96505,AAAAAAAA78938} - 64 | {26,19,34,24,81,78} | {A96617,AAAAAAAAAAAAAAAAAAA70104,A68938,AAAAAAAAAAA53908,AAAAAAAAAAAAAAA453,AA17009,AAAAAAA80240} - 65 | {61,5,76,59,17} | {AAAAAA99807,AAAAA64741,AAAAAAAAAAA53908,AA21643,AAAAAAAAA10012} - 66 | {31,23,70,52,4,33,48,25} | {AAAAAAAAAAAAAAAAA69675,AAAAAAAA50094,AAAAAAAAAAA92631,AAAA35194,39557,AAAAAAA99836} - 67 | {31,94,7,10} | {AAAAAA38063,A96617,AAAA35194,AAAAAAAAAAAA67946} - 68 | {90,43,38} | {AA75092,AAAAAAAAAAAAAAAAA69675,AAAAAAAAAAA92631,AAAAAAAAA10012,AAAAAAAAAAAAA7929,AA21643} - 69 | {67,35,99,85,72,86,44} | {AAAAAAAAAAAAAAAAAAA1205,AAAAAAAA50094,AAAAAAAAAAAAAAAA1729,AAAAAAAAAAAAAAAAAA47955} - 70 | {56,70,83} | {AAAA41702,AAAAAAAAAAA82945,AA21643,AAAAAAAAAAA99000,A27153,AA25381,AAAAAAAAAAAAAA96505,AAAAAAA1242} - 71 | {74,26} | {AAAAAAAAAAA50956,AA74433,AAAAAAA21462,AAAAAAAAAAAAAAAAAAA17075,AAAAAAAAAAAAAAA36627,AAAAAAAAAAAAA70254,AAAAAAAAAA43419,39557} - 72 | {22,1,16,78,20,91,83} | {47735,AAAAAAA56483,AAAAAAAAAAAAA93788,AA42406,AAAAAAAAAAAAA73084,AAAAAAAA72908,AAAAAAAAAAAAAAAAAA61286,AAAAA66674,AAAAAAAAAAAAAAAAA50407} - 73 | {88,25,96,78,65,15,29,19} | {AAA54451,AAAAAAAAA27249,AAAAAAA9228,AAAAAAAAAAAAAAA67062,AAAAAAAAAAAAAAAAAAA70415,AAAAA17383,AAAAAAAAAAAAAAAA33598} - 74 | {32} | {AAAAAAAAAAAAAAAA1729,AAAAAAAAAAAAA22860,AAAAAA99807,AAAAA17383,AAAAAAAAAAAAAAA67062,AAAAAAAAAAA15165,AAAAAAAAAAA50956} - 75 | {12,96,83,24,71,89,55} | {AAAA48949,AAAAAAAA29716,AAAAAAAAAAAAAAAAAAA1205,AAAAAAAAAAAA67946,AAAAAAAAAAAAAAAA29150,AAA28075,AAAAAAAAAAAAAAAAA43052} - 76 | {92,55,10,7} | {AAAAAAAAAAAAAAA67062} - 77 | {97,15,32,17,55,59,18,37,50,39} | {AAAAAAAAAAAA67946,AAAAAA54032,AAAAAAAA81587,55847,AAAAAAAAAAAAAA28620,AAAAAAAAAAAAAAAAA43052,AAAAAA75463,AAAA49534,AAAAAAAA44066} - 78 | {55,89,44,84,34} | {AAAAAAAAAAA6119,AAAAAAAAAAAAAA8666,AA99927,AA42406,AAAAAAA81898,AAAAAAA9228,AAAAAAAAAAA92631,AA21643,AAAAAAAAAAAAAA28620} - 79 | {45} | {AAAAAAAAAA646,AAAAAAAAAAAAAAAAAAA70415,AAAAAA43678,AAAAAAAA72908} - 80 | {74,89,44,80,0} | {AAAA35194,AAAAAAAA79710,AAA20874,AAAAAAAAAAAAAAAAAAA70104,AAAAAAAAAAAAA73084,AAAAAAA57334,AAAAAAA9228,AAAAAAAAAAAAA62007} - 81 | {63,77,54,48,61,53,97} | {AAAAAAAAAAAAAAA81326,AAAAAAAAAA22292,AA25381,AAAAAAAAAAA74076,AAAAAAA81898,AAAAAAAAA72121} - 82 | {34,60,4,79,78,16,86,89,42,50} | {AAAAA40681,AAAAAAAAAAAAAAAAAA12591,AAAAAAA80240,AAAAAAAAAAAAAAAA55798,AAAAAAAAAAAAAAAAAAA70104} - 83 | {14,10} | {AAAAAAAAAA22292,AAAAAAAAAAAAA70254,AAAAAAAAAAA6119} - 84 | {11,83,35,13,96,94} | {AAAAA95309,AAAAAAAAAAAAAAAAAA32918,AAAAAAAAAAAAAAAAAA24183} - 85 | {39,60} | {AAAAAAAAAAAAAAAA55798,AAAAAAAAAA22292,AAAAAAA66161,AAAAAAA21462,AAAAAAAAAAAAAAAAAA12591,55847,AAAAAA98232,AAAAAAAAAAA46154} - 86 | {33,81,72,74,45,36,82} | {AAAAAAAA81587,AAAAAAAAAAAAAA96505,45449,AAAA80176} - 87 | {57,27,50,12,97,68} | {AAAAAAAAAAAAAAAAA26540,AAAAAAAAA10012,AAAAAAAAAAAA35809,AAAAAAAAAAAAAAAA29150,AAAAAAAAAAA82945,AAAAAA66777,31228,AAAAAAAAAAAAAAAA23657,AAAAAAAAAAAAAA28620,AAAAAAAAAAAAAA96505} - 88 | {41,90,77,24,6,24} | {AAAA35194,AAAA35194,AAAAAAA80240,AAAAAAAAAAA46154,AAAAAA58494,AAAAAAAAAAAAAAAAAAA17075,AAAAAAAAAAAAAAAAAA59334,AAAAAAAAAAAAAAAAAAA91804,AA74433} - 89 | {40,32,17,6,30,88} | {AA44673,AAAAAAAAAAA6119,AAAAAAAAAAAAAAAA23657,AAAAAAAAAAAAAAAAAA47955,AAAAAAAAAAAAAAAA33598,AAAAAAAAAAA33576,AA44673} - 90 | {88,75} | {AAAAA60038,AAAAAAAA23648,AAAAAAAAAAA99000,AAAA41702,AAAAAAAAAAAAA22860,AAAAAAAAAAAAAAA68526} - 91 | {78} | {AAAAAAAAAAAAA62007,AAA99043} - 92 | {85,63,49,45} | {AAAAAAA89932,AAAAAAAAAAAAA22860,AAAAAAAAAAAAAAAAAAA1205,AAAAAAAAAAAA21089} - 93 | {11} | {AAAAAAAAAAA176,AAAAAAAAAAAAAA8666,AAAAAAAAAAAAAAA453,AAAAAAAAAAAAA85723,A68938,AAAAAAAAAAAAA9821,AAAAAAA48038,AAAAAAAAAAAAAAAAA59387,AA99927,AAAAA17383} - 94 | {98,9,85,62,88,91,60,61,38,86} | {AAAAAAAA81587,AAAAA17383,AAAAAAAA81587} - 95 | {47,77} | {AAAAAAAAAAAAAAAAA764,AAAAAAAAAAA74076,AAAAAAAAAA18107,AAAAA40681,AAAAAAAAAAAAAAA35875,AAAAA60038,AAAAAAA56483} - 96 | {23,97,43} | {AAAAAAAAAA646,A87088} - 97 | {54,2,86,65} | {47735,AAAAAAA99836,AAAAAAAAAAAAAAAAA6897,AAAAAAAAAAAAAAAA29150,AAAAAAA80240,AAAAAAAAAAAAAAAA98414,AAAAAAA56483,AAAAAAAAAAAAAAAA29150,AAAAAAA39692,AA21643} - 98 | {38,34,32,89} | {AAAAAAAAAAAAAAAAAA71621,AAAA8857,AAAAAAAAAAAAAAAAAAA65037,AAAAAAAAAAAAAAAA31334,AAAAAAAAAA48845} - 99 | {37,86} | {AAAAAAAAAAAAAAAAAA32918,AAAAA70514,AAAAAAAAA10012,AAAAAAAAAAAAAAAAA59387,AAAAAAAAAA64777,AAAAAAAAAAAAAAAAAAA15356} - 100 | {85,32,57,39,49,84,32,3,30} | {AAAAAAA80240,AAAAAAAAAAAAAAAA1729,AAAAA60038,AAAAAAAAAAA92631,AAAAAAAA9523} - 101 | {} | {} - 102 | {NULL} | {NULL} -(102 rows) - -SELECT * FROM array_index_op_test WHERE t && '{}' ORDER BY seqno; - seqno | i | t --------+---+--- -(0 rows) - -SELECT * FROM array_index_op_test WHERE t <@ '{}' ORDER BY seqno; - seqno | i | t --------+----+---- - 101 | {} | {} -(1 row) - --- And try it with a multicolumn GIN index -DROP INDEX intarrayidx, textarrayidx; -CREATE INDEX botharrayidx ON array_index_op_test USING gin (i, t); -SELECT * FROM array_index_op_test WHERE i @> '{32}' ORDER BY seqno; - seqno | i | t --------+---------------------------------+------------------------------------------------------------------------------------------------------------------------------------ - 6 | {39,35,5,94,17,92,60,32} | {AAAAAAAAAAAAAAA35875,AAAAAAAAAAAAAAAA23657} - 74 | {32} | {AAAAAAAAAAAAAAAA1729,AAAAAAAAAAAAA22860,AAAAAA99807,AAAAA17383,AAAAAAAAAAAAAAA67062,AAAAAAAAAAA15165,AAAAAAAAAAA50956} - 77 | {97,15,32,17,55,59,18,37,50,39} | {AAAAAAAAAAAA67946,AAAAAA54032,AAAAAAAA81587,55847,AAAAAAAAAAAAAA28620,AAAAAAAAAAAAAAAAA43052,AAAAAA75463,AAAA49534,AAAAAAAA44066} - 89 | {40,32,17,6,30,88} | {AA44673,AAAAAAAAAAA6119,AAAAAAAAAAAAAAAA23657,AAAAAAAAAAAAAAAAAA47955,AAAAAAAAAAAAAAAA33598,AAAAAAAAAAA33576,AA44673} - 98 | {38,34,32,89} | {AAAAAAAAAAAAAAAAAA71621,AAAA8857,AAAAAAAAAAAAAAAAAAA65037,AAAAAAAAAAAAAAAA31334,AAAAAAAAAA48845} - 100 | {85,32,57,39,49,84,32,3,30} | {AAAAAAA80240,AAAAAAAAAAAAAAAA1729,AAAAA60038,AAAAAAAAAAA92631,AAAAAAAA9523} -(6 rows) - -SELECT * FROM array_index_op_test WHERE i && '{32}' ORDER BY seqno; - seqno | i | t --------+---------------------------------+------------------------------------------------------------------------------------------------------------------------------------ - 6 | {39,35,5,94,17,92,60,32} | {AAAAAAAAAAAAAAA35875,AAAAAAAAAAAAAAAA23657} - 74 | {32} | {AAAAAAAAAAAAAAAA1729,AAAAAAAAAAAAA22860,AAAAAA99807,AAAAA17383,AAAAAAAAAAAAAAA67062,AAAAAAAAAAA15165,AAAAAAAAAAA50956} - 77 | {97,15,32,17,55,59,18,37,50,39} | {AAAAAAAAAAAA67946,AAAAAA54032,AAAAAAAA81587,55847,AAAAAAAAAAAAAA28620,AAAAAAAAAAAAAAAAA43052,AAAAAA75463,AAAA49534,AAAAAAAA44066} - 89 | {40,32,17,6,30,88} | {AA44673,AAAAAAAAAAA6119,AAAAAAAAAAAAAAAA23657,AAAAAAAAAAAAAAAAAA47955,AAAAAAAAAAAAAAAA33598,AAAAAAAAAAA33576,AA44673} - 98 | {38,34,32,89} | {AAAAAAAAAAAAAAAAAA71621,AAAA8857,AAAAAAAAAAAAAAAAAAA65037,AAAAAAAAAAAAAAAA31334,AAAAAAAAAA48845} - 100 | {85,32,57,39,49,84,32,3,30} | {AAAAAAA80240,AAAAAAAAAAAAAAAA1729,AAAAA60038,AAAAAAAAAAA92631,AAAAAAAA9523} -(6 rows) - -SELECT * FROM array_index_op_test WHERE t @> '{AAAAAAA80240}' ORDER BY seqno; - seqno | i | t --------+--------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------- - 19 | {52,82,17,74,23,46,69,51,75} | {AAAAAAAAAAAAA73084,AAAAA75968,AAAAAAAAAAAAAAAA14047,AAAAAAA80240,AAAAAAAAAAAAAAAAAAA1205,A68938} - 30 | {26,81,47,91,34} | {AAAAAAAAAAAAAAAAAAA70104,AAAAAAA80240} - 64 | {26,19,34,24,81,78} | {A96617,AAAAAAAAAAAAAAAAAAA70104,A68938,AAAAAAAAAAA53908,AAAAAAAAAAAAAAA453,AA17009,AAAAAAA80240} - 82 | {34,60,4,79,78,16,86,89,42,50} | {AAAAA40681,AAAAAAAAAAAAAAAAAA12591,AAAAAAA80240,AAAAAAAAAAAAAAAA55798,AAAAAAAAAAAAAAAAAAA70104} - 88 | {41,90,77,24,6,24} | {AAAA35194,AAAA35194,AAAAAAA80240,AAAAAAAAAAA46154,AAAAAA58494,AAAAAAAAAAAAAAAAAAA17075,AAAAAAAAAAAAAAAAAA59334,AAAAAAAAAAAAAAAAAAA91804,AA74433} - 97 | {54,2,86,65} | {47735,AAAAAAA99836,AAAAAAAAAAAAAAAAA6897,AAAAAAAAAAAAAAAA29150,AAAAAAA80240,AAAAAAAAAAAAAAAA98414,AAAAAAA56483,AAAAAAAAAAAAAAAA29150,AAAAAAA39692,AA21643} - 100 | {85,32,57,39,49,84,32,3,30} | {AAAAAAA80240,AAAAAAAAAAAAAAAA1729,AAAAA60038,AAAAAAAAAAA92631,AAAAAAAA9523} -(7 rows) - -SELECT * FROM array_index_op_test WHERE t && '{AAAAAAA80240}' ORDER BY seqno; - seqno | i | t --------+--------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------- - 19 | {52,82,17,74,23,46,69,51,75} | {AAAAAAAAAAAAA73084,AAAAA75968,AAAAAAAAAAAAAAAA14047,AAAAAAA80240,AAAAAAAAAAAAAAAAAAA1205,A68938} - 30 | {26,81,47,91,34} | {AAAAAAAAAAAAAAAAAAA70104,AAAAAAA80240} - 64 | {26,19,34,24,81,78} | {A96617,AAAAAAAAAAAAAAAAAAA70104,A68938,AAAAAAAAAAA53908,AAAAAAAAAAAAAAA453,AA17009,AAAAAAA80240} - 82 | {34,60,4,79,78,16,86,89,42,50} | {AAAAA40681,AAAAAAAAAAAAAAAAAA12591,AAAAAAA80240,AAAAAAAAAAAAAAAA55798,AAAAAAAAAAAAAAAAAAA70104} - 88 | {41,90,77,24,6,24} | {AAAA35194,AAAA35194,AAAAAAA80240,AAAAAAAAAAA46154,AAAAAA58494,AAAAAAAAAAAAAAAAAAA17075,AAAAAAAAAAAAAAAAAA59334,AAAAAAAAAAAAAAAAAAA91804,AA74433} - 97 | {54,2,86,65} | {47735,AAAAAAA99836,AAAAAAAAAAAAAAAAA6897,AAAAAAAAAAAAAAAA29150,AAAAAAA80240,AAAAAAAAAAAAAAAA98414,AAAAAAA56483,AAAAAAAAAAAAAAAA29150,AAAAAAA39692,AA21643} - 100 | {85,32,57,39,49,84,32,3,30} | {AAAAAAA80240,AAAAAAAAAAAAAAAA1729,AAAAA60038,AAAAAAAAAAA92631,AAAAAAAA9523} -(7 rows) - -SELECT * FROM array_index_op_test WHERE i @> '{32}' AND t && '{AAAAAAA80240}' ORDER BY seqno; - seqno | i | t --------+-----------------------------+------------------------------------------------------------------------------ - 100 | {85,32,57,39,49,84,32,3,30} | {AAAAAAA80240,AAAAAAAAAAAAAAAA1729,AAAAA60038,AAAAAAAAAAA92631,AAAAAAAA9523} -(1 row) - -SELECT * FROM array_index_op_test WHERE i && '{32}' AND t @> '{AAAAAAA80240}' ORDER BY seqno; - seqno | i | t --------+-----------------------------+------------------------------------------------------------------------------ - 100 | {85,32,57,39,49,84,32,3,30} | {AAAAAAA80240,AAAAAAAAAAAAAAAA1729,AAAAA60038,AAAAAAAAAAA92631,AAAAAAAA9523} -(1 row) - -SELECT * FROM array_index_op_test WHERE t = '{}' ORDER BY seqno; - seqno | i | t --------+----+---- - 101 | {} | {} -(1 row) - -RESET enable_seqscan; -RESET enable_indexscan; -RESET enable_bitmapscan; --- --- Try a GIN index with a lot of items with same key. (GIN creates a posting --- tree when there are enough duplicates) --- -CREATE TABLE array_gin_test (a int[]); -INSERT INTO array_gin_test SELECT ARRAY[1, g%5, g] FROM generate_series(1, 10000) g; -CREATE INDEX array_gin_test_idx ON array_gin_test USING gin (a); -SELECT COUNT(*) FROM array_gin_test WHERE a @> '{2}'; - count -------- - 2000 -(1 row) - -DROP TABLE array_gin_test; --- --- Test GIN index's reloptions --- -CREATE INDEX gin_relopts_test ON array_index_op_test USING gin (i) - WITH (FASTUPDATE=on, GIN_PENDING_LIST_LIMIT=128); -\d+ gin_relopts_test - Index "public.gin_relopts_test" - Column | Type | Key? | Definition | Storage | Stats target ---------+---------+------+------------+---------+-------------- - i | integer | yes | i | plain | -gin, for table "public.array_index_op_test" -Options: fastupdate=on, gin_pending_list_limit=128 - --- --- HASH --- -CREATE UNLOGGED TABLE unlogged_hash_table (id int4); -CREATE INDEX unlogged_hash_index ON unlogged_hash_table USING hash (id int4_ops); -DROP TABLE unlogged_hash_table; --- CREATE INDEX hash_ovfl_index ON hash_ovfl_heap USING hash (x int4_ops); --- Test hash index build tuplesorting. Force hash tuplesort using low --- maintenance_work_mem setting and fillfactor: -SET maintenance_work_mem = '1MB'; -CREATE INDEX hash_tuplesort_idx ON tenk1 USING hash (stringu1 name_ops) WITH (fillfactor = 10); -EXPLAIN (COSTS OFF) -SELECT count(*) FROM tenk1 WHERE stringu1 = 'TVAAAA'; - QUERY PLAN -------------------------------------------------------- - Aggregate - -> Bitmap Heap Scan on tenk1 - Recheck Cond: (stringu1 = 'TVAAAA'::name) - -> Bitmap Index Scan on hash_tuplesort_idx - Index Cond: (stringu1 = 'TVAAAA'::name) -(5 rows) - -SELECT count(*) FROM tenk1 WHERE stringu1 = 'TVAAAA'; - count -------- - 14 -(1 row) - -DROP INDEX hash_tuplesort_idx; -RESET maintenance_work_mem; --- --- Test unique null behavior --- -CREATE TABLE unique_tbl (i int, t text); -CREATE UNIQUE INDEX unique_idx1 ON unique_tbl (i) NULLS DISTINCT; -CREATE UNIQUE INDEX unique_idx2 ON unique_tbl (i) NULLS NOT DISTINCT; -INSERT INTO unique_tbl VALUES (1, 'one'); -INSERT INTO unique_tbl VALUES (2, 'two'); -INSERT INTO unique_tbl VALUES (3, 'three'); -INSERT INTO unique_tbl VALUES (4, 'four'); -INSERT INTO unique_tbl VALUES (5, 'one'); -INSERT INTO unique_tbl (t) VALUES ('six'); -INSERT INTO unique_tbl (t) VALUES ('seven'); -- error from unique_idx2 -ERROR: duplicate key value violates unique constraint "unique_idx2" -DETAIL: Key (i)=(null) already exists. -DROP INDEX unique_idx1, unique_idx2; -INSERT INTO unique_tbl (t) VALUES ('seven'); --- build indexes on filled table -CREATE UNIQUE INDEX unique_idx3 ON unique_tbl (i) NULLS DISTINCT; -- ok -CREATE UNIQUE INDEX unique_idx4 ON unique_tbl (i) NULLS NOT DISTINCT; -- error -ERROR: could not create unique index "unique_idx4" -DETAIL: Key (i)=(null) is duplicated. -DELETE FROM unique_tbl WHERE t = 'seven'; -CREATE UNIQUE INDEX unique_idx4 ON unique_tbl (i) NULLS NOT DISTINCT; -- ok now -\d unique_tbl - Table "public.unique_tbl" - Column | Type | Collation | Nullable | Default ---------+---------+-----------+----------+--------- - i | integer | | | - t | text | | | -Indexes: - "unique_idx3" UNIQUE, btree (i) - "unique_idx4" UNIQUE, btree (i) NULLS NOT DISTINCT - -\d unique_idx3 - Index "public.unique_idx3" - Column | Type | Key? | Definition ---------+---------+------+------------ - i | integer | yes | i -unique, btree, for table "public.unique_tbl" - -\d unique_idx4 - Index "public.unique_idx4" - Column | Type | Key? | Definition ---------+---------+------+------------ - i | integer | yes | i -unique nulls not distinct, btree, for table "public.unique_tbl" - -SELECT pg_get_indexdef('unique_idx3'::regclass); - pg_get_indexdef ----------------------------------------------------------------------- - CREATE UNIQUE INDEX unique_idx3 ON public.unique_tbl USING btree (i) -(1 row) - -SELECT pg_get_indexdef('unique_idx4'::regclass); - pg_get_indexdef ------------------------------------------------------------------------------------------ - CREATE UNIQUE INDEX unique_idx4 ON public.unique_tbl USING btree (i) NULLS NOT DISTINCT -(1 row) - -DROP TABLE unique_tbl; --- --- Test functional index --- -CREATE TABLE func_index_heap (f1 text, f2 text); -CREATE UNIQUE INDEX func_index_index on func_index_heap (textcat(f1,f2)); -INSERT INTO func_index_heap VALUES('ABC','DEF'); -INSERT INTO func_index_heap VALUES('AB','CDEFG'); -INSERT INTO func_index_heap VALUES('QWE','RTY'); --- this should fail because of unique index: -INSERT INTO func_index_heap VALUES('ABCD', 'EF'); -ERROR: duplicate key value violates unique constraint "func_index_index" -DETAIL: Key (textcat(f1, f2))=(ABCDEF) already exists. --- but this shouldn't: -INSERT INTO func_index_heap VALUES('QWERTY'); --- while we're here, see that the metadata looks sane -\d func_index_heap - Table "public.func_index_heap" - Column | Type | Collation | Nullable | Default ---------+------+-----------+----------+--------- - f1 | text | | | - f2 | text | | | -Indexes: - "func_index_index" UNIQUE, btree (textcat(f1, f2)) - -\d func_index_index - Index "public.func_index_index" - Column | Type | Key? | Definition ----------+------+------+----------------- - textcat | text | yes | textcat(f1, f2) -unique, btree, for table "public.func_index_heap" - --- --- Same test, expressional index --- -DROP TABLE func_index_heap; -CREATE TABLE func_index_heap (f1 text, f2 text); -CREATE UNIQUE INDEX func_index_index on func_index_heap ((f1 || f2) text_ops); -INSERT INTO func_index_heap VALUES('ABC','DEF'); -INSERT INTO func_index_heap VALUES('AB','CDEFG'); -INSERT INTO func_index_heap VALUES('QWE','RTY'); --- this should fail because of unique index: -INSERT INTO func_index_heap VALUES('ABCD', 'EF'); -ERROR: duplicate key value violates unique constraint "func_index_index" -DETAIL: Key ((f1 || f2))=(ABCDEF) already exists. --- but this shouldn't: -INSERT INTO func_index_heap VALUES('QWERTY'); --- while we're here, see that the metadata looks sane -\d func_index_heap - Table "public.func_index_heap" - Column | Type | Collation | Nullable | Default ---------+------+-----------+----------+--------- - f1 | text | | | - f2 | text | | | -Indexes: - "func_index_index" UNIQUE, btree ((f1 || f2)) - -\d func_index_index - Index "public.func_index_index" - Column | Type | Key? | Definition ---------+------+------+------------ - expr | text | yes | (f1 || f2) -unique, btree, for table "public.func_index_heap" - --- this should fail because of unsafe column type (anonymous record) -create index on func_index_heap ((f1 || f2), (row(f1, f2))); -ERROR: column "row" has pseudo-type record --- --- Test unique index with included columns --- -CREATE TABLE covering_index_heap (f1 int, f2 int, f3 text); -CREATE UNIQUE INDEX covering_index_index on covering_index_heap (f1,f2) INCLUDE(f3); -INSERT INTO covering_index_heap VALUES(1,1,'AAA'); -INSERT INTO covering_index_heap VALUES(1,2,'AAA'); --- this should fail because of unique index on f1,f2: -INSERT INTO covering_index_heap VALUES(1,2,'BBB'); -ERROR: duplicate key value violates unique constraint "covering_index_index" -DETAIL: Key (f1, f2)=(1, 2) already exists. --- and this shouldn't: -INSERT INTO covering_index_heap VALUES(1,4,'AAA'); --- Try to build index on table that already contains data -CREATE UNIQUE INDEX covering_pkey on covering_index_heap (f1,f2) INCLUDE(f3); --- Try to use existing covering index as primary key -ALTER TABLE covering_index_heap ADD CONSTRAINT covering_pkey PRIMARY KEY USING INDEX -covering_pkey; -DROP TABLE covering_index_heap; --- --- Try some concurrent index builds --- --- Unfortunately this only tests about half the code paths because there are --- no concurrent updates happening to the table at the same time. -CREATE TABLE concur_heap (f1 text, f2 text); --- empty table -CREATE INDEX CONCURRENTLY concur_index1 ON concur_heap(f2,f1); -CREATE INDEX CONCURRENTLY IF NOT EXISTS concur_index1 ON concur_heap(f2,f1); -NOTICE: relation "concur_index1" already exists, skipping -INSERT INTO concur_heap VALUES ('a','b'); -INSERT INTO concur_heap VALUES ('b','b'); --- unique index -CREATE UNIQUE INDEX CONCURRENTLY concur_index2 ON concur_heap(f1); -CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS concur_index2 ON concur_heap(f1); -NOTICE: relation "concur_index2" already exists, skipping --- check if constraint is set up properly to be enforced -INSERT INTO concur_heap VALUES ('b','x'); -ERROR: duplicate key value violates unique constraint "concur_index2" -DETAIL: Key (f1)=(b) already exists. --- check if constraint is enforced properly at build time -CREATE UNIQUE INDEX CONCURRENTLY concur_index3 ON concur_heap(f2); -ERROR: could not create unique index "concur_index3" -DETAIL: Key (f2)=(b) is duplicated. --- test that expression indexes and partial indexes work concurrently -CREATE INDEX CONCURRENTLY concur_index4 on concur_heap(f2) WHERE f1='a'; -CREATE INDEX CONCURRENTLY concur_index5 on concur_heap(f2) WHERE f1='x'; --- here we also check that you can default the index name -CREATE INDEX CONCURRENTLY on concur_heap((f2||f1)); --- You can't do a concurrent index build in a transaction -BEGIN; -CREATE INDEX CONCURRENTLY concur_index7 ON concur_heap(f1); -ERROR: CREATE INDEX CONCURRENTLY cannot run inside a transaction block -COMMIT; --- test where predicate is able to do a transactional update during --- a concurrent build before switching pg_index state flags. -CREATE FUNCTION predicate_stable() RETURNS bool IMMUTABLE -LANGUAGE plpgsql AS $$ -BEGIN - EXECUTE 'SELECT txid_current()'; - RETURN true; -END; $$; -CREATE INDEX CONCURRENTLY concur_index8 ON concur_heap (f1) - WHERE predicate_stable(); -DROP INDEX concur_index8; -DROP FUNCTION predicate_stable(); --- But you can do a regular index build in a transaction -BEGIN; -CREATE INDEX std_index on concur_heap(f2); -COMMIT; --- Failed builds are left invalid by VACUUM FULL, fixed by REINDEX -VACUUM FULL concur_heap; -REINDEX TABLE concur_heap; -ERROR: could not create unique index "concur_index3" -DETAIL: Key (f2)=(b) is duplicated. -DELETE FROM concur_heap WHERE f1 = 'b'; -VACUUM FULL concur_heap; -\d concur_heap - Table "public.concur_heap" - Column | Type | Collation | Nullable | Default ---------+------+-----------+----------+--------- - f1 | text | | | - f2 | text | | | -Indexes: - "concur_heap_expr_idx" btree ((f2 || f1)) - "concur_index1" btree (f2, f1) - "concur_index2" UNIQUE, btree (f1) - "concur_index3" UNIQUE, btree (f2) INVALID - "concur_index4" btree (f2) WHERE f1 = 'a'::text - "concur_index5" btree (f2) WHERE f1 = 'x'::text - "std_index" btree (f2) - -REINDEX TABLE concur_heap; -\d concur_heap - Table "public.concur_heap" - Column | Type | Collation | Nullable | Default ---------+------+-----------+----------+--------- - f1 | text | | | - f2 | text | | | -Indexes: - "concur_heap_expr_idx" btree ((f2 || f1)) - "concur_index1" btree (f2, f1) - "concur_index2" UNIQUE, btree (f1) - "concur_index3" UNIQUE, btree (f2) - "concur_index4" btree (f2) WHERE f1 = 'a'::text - "concur_index5" btree (f2) WHERE f1 = 'x'::text - "std_index" btree (f2) - --- Temporary tables with concurrent builds and on-commit actions --- CONCURRENTLY used with CREATE INDEX and DROP INDEX is ignored. --- PRESERVE ROWS, the default. -CREATE TEMP TABLE concur_temp (f1 int, f2 text) - ON COMMIT PRESERVE ROWS; -INSERT INTO concur_temp VALUES (1, 'foo'), (2, 'bar'); -CREATE INDEX CONCURRENTLY concur_temp_ind ON concur_temp(f1); -DROP INDEX CONCURRENTLY concur_temp_ind; -DROP TABLE concur_temp; --- ON COMMIT DROP -BEGIN; -CREATE TEMP TABLE concur_temp (f1 int, f2 text) - ON COMMIT DROP; -INSERT INTO concur_temp VALUES (1, 'foo'), (2, 'bar'); --- Fails when running in a transaction. -CREATE INDEX CONCURRENTLY concur_temp_ind ON concur_temp(f1); -ERROR: CREATE INDEX CONCURRENTLY cannot run inside a transaction block -COMMIT; --- ON COMMIT DELETE ROWS -CREATE TEMP TABLE concur_temp (f1 int, f2 text) - ON COMMIT DELETE ROWS; -INSERT INTO concur_temp VALUES (1, 'foo'), (2, 'bar'); -CREATE INDEX CONCURRENTLY concur_temp_ind ON concur_temp(f1); -DROP INDEX CONCURRENTLY concur_temp_ind; -DROP TABLE concur_temp; --- --- Try some concurrent index drops --- -DROP INDEX CONCURRENTLY "concur_index2"; -- works -DROP INDEX CONCURRENTLY IF EXISTS "concur_index2"; -- notice -NOTICE: index "concur_index2" does not exist, skipping --- failures -DROP INDEX CONCURRENTLY "concur_index2", "concur_index3"; -ERROR: DROP INDEX CONCURRENTLY does not support dropping multiple objects -BEGIN; -DROP INDEX CONCURRENTLY "concur_index5"; -ERROR: DROP INDEX CONCURRENTLY cannot run inside a transaction block -ROLLBACK; --- successes -DROP INDEX CONCURRENTLY IF EXISTS "concur_index3"; -DROP INDEX CONCURRENTLY "concur_index4"; -DROP INDEX CONCURRENTLY "concur_index5"; -DROP INDEX CONCURRENTLY "concur_index1"; -DROP INDEX CONCURRENTLY "concur_heap_expr_idx"; -\d concur_heap - Table "public.concur_heap" - Column | Type | Collation | Nullable | Default ---------+------+-----------+----------+--------- - f1 | text | | | - f2 | text | | | -Indexes: - "std_index" btree (f2) - -DROP TABLE concur_heap; --- --- Test ADD CONSTRAINT USING INDEX --- -CREATE TABLE cwi_test( a int , b varchar(10), c char); --- add some data so that all tests have something to work with. -INSERT INTO cwi_test VALUES(1, 2), (3, 4), (5, 6); -CREATE UNIQUE INDEX cwi_uniq_idx ON cwi_test(a , b); -ALTER TABLE cwi_test ADD primary key USING INDEX cwi_uniq_idx; -\d cwi_test - Table "public.cwi_test" - Column | Type | Collation | Nullable | Default ---------+-----------------------+-----------+----------+--------- - a | integer | | not null | - b | character varying(10) | | not null | - c | character(1) | | | -Indexes: - "cwi_uniq_idx" PRIMARY KEY, btree (a, b) - -\d cwi_uniq_idx - Index "public.cwi_uniq_idx" - Column | Type | Key? | Definition ---------+-----------------------+------+------------ - a | integer | yes | a - b | character varying(10) | yes | b -primary key, btree, for table "public.cwi_test" - -CREATE UNIQUE INDEX cwi_uniq2_idx ON cwi_test(b , a); -ALTER TABLE cwi_test DROP CONSTRAINT cwi_uniq_idx, - ADD CONSTRAINT cwi_replaced_pkey PRIMARY KEY - USING INDEX cwi_uniq2_idx; -NOTICE: ALTER TABLE / ADD CONSTRAINT USING INDEX will rename index "cwi_uniq2_idx" to "cwi_replaced_pkey" -\d cwi_test - Table "public.cwi_test" - Column | Type | Collation | Nullable | Default ---------+-----------------------+-----------+----------+--------- - a | integer | | not null | - b | character varying(10) | | not null | - c | character(1) | | | -Indexes: - "cwi_replaced_pkey" PRIMARY KEY, btree (b, a) - -\d cwi_replaced_pkey - Index "public.cwi_replaced_pkey" - Column | Type | Key? | Definition ---------+-----------------------+------+------------ - b | character varying(10) | yes | b - a | integer | yes | a -primary key, btree, for table "public.cwi_test" - -DROP INDEX cwi_replaced_pkey; -- Should fail; a constraint depends on it -ERROR: cannot drop index cwi_replaced_pkey because constraint cwi_replaced_pkey on table cwi_test requires it -HINT: You can drop constraint cwi_replaced_pkey on table cwi_test instead. --- Check that non-default index options are rejected -CREATE UNIQUE INDEX cwi_uniq3_idx ON cwi_test(a desc); -ALTER TABLE cwi_test ADD UNIQUE USING INDEX cwi_uniq3_idx; -- fail -ERROR: index "cwi_uniq3_idx" column number 1 does not have default sorting behavior -LINE 1: ALTER TABLE cwi_test ADD UNIQUE USING INDEX cwi_uniq3_idx; - ^ -DETAIL: Cannot create a primary key or unique constraint using such an index. -CREATE UNIQUE INDEX cwi_uniq4_idx ON cwi_test(b collate "POSIX"); -ALTER TABLE cwi_test ADD UNIQUE USING INDEX cwi_uniq4_idx; -- fail -ERROR: index "cwi_uniq4_idx" column number 1 does not have default sorting behavior -LINE 1: ALTER TABLE cwi_test ADD UNIQUE USING INDEX cwi_uniq4_idx; - ^ -DETAIL: Cannot create a primary key or unique constraint using such an index. -DROP TABLE cwi_test; --- ADD CONSTRAINT USING INDEX is forbidden on partitioned tables -CREATE TABLE cwi_test(a int) PARTITION BY hash (a); -create unique index on cwi_test (a); -alter table cwi_test add primary key using index cwi_test_a_idx ; -ERROR: ALTER TABLE / ADD CONSTRAINT USING INDEX is not supported on partitioned tables -DROP TABLE cwi_test; --- PRIMARY KEY constraint cannot be backed by a NULLS NOT DISTINCT index -CREATE TABLE cwi_test(a int, b int); -CREATE UNIQUE INDEX cwi_a_nnd ON cwi_test (a) NULLS NOT DISTINCT; -ALTER TABLE cwi_test ADD PRIMARY KEY USING INDEX cwi_a_nnd; -ERROR: primary keys cannot use NULLS NOT DISTINCT indexes -DROP TABLE cwi_test; --- --- Check handling of indexes on system columns --- -CREATE TABLE syscol_table (a INT); --- System columns cannot be indexed -CREATE INDEX ON syscolcol_table (ctid); -ERROR: relation "syscolcol_table" does not exist --- nor used in expressions -CREATE INDEX ON syscol_table ((ctid >= '(1000,0)')); -ERROR: index creation on system columns is not supported --- nor used in predicates -CREATE INDEX ON syscol_table (a) WHERE ctid >= '(1000,0)'; -ERROR: index creation on system columns is not supported -DROP TABLE syscol_table; --- --- Tests for IS NULL/IS NOT NULL with b-tree indexes --- -CREATE TABLE onek_with_null AS SELECT unique1, unique2 FROM onek; -INSERT INTO onek_with_null (unique1,unique2) VALUES (NULL, -1), (NULL, NULL); -CREATE UNIQUE INDEX onek_nulltest ON onek_with_null (unique2,unique1); -SET enable_seqscan = OFF; -SET enable_indexscan = ON; -SET enable_bitmapscan = ON; -SELECT count(*) FROM onek_with_null WHERE unique1 IS NULL; - count -------- - 2 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NULL AND unique2 IS NULL; - count -------- - 1 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NOT NULL; - count -------- - 1000 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NULL AND unique2 IS NOT NULL; - count -------- - 1 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NOT NULL AND unique1 > 500; - count -------- - 499 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NULL AND unique1 > 500; - count -------- - 0 -(1 row) - -DROP INDEX onek_nulltest; -CREATE UNIQUE INDEX onek_nulltest ON onek_with_null (unique2 desc,unique1); -SELECT count(*) FROM onek_with_null WHERE unique1 IS NULL; - count -------- - 2 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NULL AND unique2 IS NULL; - count -------- - 1 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NOT NULL; - count -------- - 1000 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NULL AND unique2 IS NOT NULL; - count -------- - 1 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NOT NULL AND unique1 > 500; - count -------- - 499 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NULL AND unique1 > 500; - count -------- - 0 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NULL AND unique2 IN (-1, 0, 1); - count -------- - 1 -(1 row) - -DROP INDEX onek_nulltest; -CREATE UNIQUE INDEX onek_nulltest ON onek_with_null (unique2 desc nulls last,unique1); -SELECT count(*) FROM onek_with_null WHERE unique1 IS NULL; - count -------- - 2 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NULL AND unique2 IS NULL; - count -------- - 1 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NOT NULL; - count -------- - 1000 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NULL AND unique2 IS NOT NULL; - count -------- - 1 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NOT NULL AND unique1 > 500; - count -------- - 499 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NULL AND unique1 > 500; - count -------- - 0 -(1 row) - -DROP INDEX onek_nulltest; -CREATE UNIQUE INDEX onek_nulltest ON onek_with_null (unique2 nulls first,unique1); -SELECT count(*) FROM onek_with_null WHERE unique1 IS NULL; - count -------- - 2 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NULL AND unique2 IS NULL; - count -------- - 1 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NOT NULL; - count -------- - 1000 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NULL AND unique2 IS NOT NULL; - count -------- - 1 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NOT NULL AND unique1 > 500; - count -------- - 499 -(1 row) - -SELECT count(*) FROM onek_with_null WHERE unique1 IS NULL AND unique1 > 500; - count -------- - 0 -(1 row) - -DROP INDEX onek_nulltest; --- Check initial-positioning logic too -CREATE UNIQUE INDEX onek_nulltest ON onek_with_null (unique2); -SET enable_seqscan = OFF; -SET enable_indexscan = ON; -SET enable_bitmapscan = OFF; -SELECT unique1, unique2 FROM onek_with_null - ORDER BY unique2 LIMIT 2; - unique1 | unique2 ----------+--------- - | -1 - 147 | 0 -(2 rows) - -SELECT unique1, unique2 FROM onek_with_null WHERE unique2 >= -1 - ORDER BY unique2 LIMIT 2; - unique1 | unique2 ----------+--------- - | -1 - 147 | 0 -(2 rows) - -SELECT unique1, unique2 FROM onek_with_null WHERE unique2 >= 0 - ORDER BY unique2 LIMIT 2; - unique1 | unique2 ----------+--------- - 147 | 0 - 931 | 1 -(2 rows) - -SELECT unique1, unique2 FROM onek_with_null - ORDER BY unique2 DESC LIMIT 2; - unique1 | unique2 ----------+--------- - | - 278 | 999 -(2 rows) - -SELECT unique1, unique2 FROM onek_with_null WHERE unique2 >= -1 - ORDER BY unique2 DESC LIMIT 2; - unique1 | unique2 ----------+--------- - 278 | 999 - 0 | 998 -(2 rows) - -SELECT unique1, unique2 FROM onek_with_null WHERE unique2 < 999 - ORDER BY unique2 DESC LIMIT 2; - unique1 | unique2 ----------+--------- - 0 | 998 - 744 | 997 -(2 rows) - -RESET enable_seqscan; -RESET enable_indexscan; -RESET enable_bitmapscan; -DROP TABLE onek_with_null; --- --- Check bitmap index path planning --- -EXPLAIN (COSTS OFF) -SELECT * FROM tenk1 - WHERE thousand = 42 AND (tenthous = 1 OR tenthous = 3 OR tenthous = 42); - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ - Bitmap Heap Scan on tenk1 - Recheck Cond: (((thousand = 42) AND (tenthous = 1)) OR ((thousand = 42) AND (tenthous = 3)) OR ((thousand = 42) AND (tenthous = 42))) - -> BitmapOr - -> Bitmap Index Scan on tenk1_thous_tenthous - Index Cond: ((thousand = 42) AND (tenthous = 1)) - -> Bitmap Index Scan on tenk1_thous_tenthous - Index Cond: ((thousand = 42) AND (tenthous = 3)) - -> Bitmap Index Scan on tenk1_thous_tenthous - Index Cond: ((thousand = 42) AND (tenthous = 42)) -(9 rows) - -SELECT * FROM tenk1 - WHERE thousand = 42 AND (tenthous = 1 OR tenthous = 3 OR tenthous = 42); - unique1 | unique2 | two | four | ten | twenty | hundred | thousand | twothousand | fivethous | tenthous | odd | even | stringu1 | stringu2 | string4 ----------+---------+-----+------+-----+--------+---------+----------+-------------+-----------+----------+-----+------+----------+----------+--------- - 42 | 5530 | 0 | 2 | 2 | 2 | 42 | 42 | 42 | 42 | 42 | 84 | 85 | QBAAAA | SEIAAA | OOOOxx -(1 row) - -EXPLAIN (COSTS OFF) -SELECT count(*) FROM tenk1 - WHERE hundred = 42 AND (thousand = 42 OR thousand = 99); - QUERY PLAN ---------------------------------------------------------------------------------- - Aggregate - -> Bitmap Heap Scan on tenk1 - Recheck Cond: ((hundred = 42) AND ((thousand = 42) OR (thousand = 99))) - -> BitmapAnd - -> Bitmap Index Scan on tenk1_hundred - Index Cond: (hundred = 42) - -> BitmapOr - -> Bitmap Index Scan on tenk1_thous_tenthous - Index Cond: (thousand = 42) - -> Bitmap Index Scan on tenk1_thous_tenthous - Index Cond: (thousand = 99) -(11 rows) - -SELECT count(*) FROM tenk1 - WHERE hundred = 42 AND (thousand = 42 OR thousand = 99); - count -------- - 10 -(1 row) - --- --- Check behavior with duplicate index column contents --- -CREATE TABLE dupindexcols AS - SELECT unique1 as id, stringu2::text as f1 FROM tenk1; -CREATE INDEX dupindexcols_i ON dupindexcols (f1, id, f1 text_pattern_ops); -ANALYZE dupindexcols; -EXPLAIN (COSTS OFF) - SELECT count(*) FROM dupindexcols - WHERE f1 BETWEEN 'WA' AND 'ZZZ' and id < 1000 and f1 ~<~ 'YX'; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------- - Aggregate - -> Bitmap Heap Scan on dupindexcols - Recheck Cond: ((f1 >= 'WA'::text) AND (f1 <= 'ZZZ'::text) AND (id < 1000) AND (f1 ~<~ 'YX'::text)) - -> Bitmap Index Scan on dupindexcols_i - Index Cond: ((f1 >= 'WA'::text) AND (f1 <= 'ZZZ'::text) AND (id < 1000) AND (f1 ~<~ 'YX'::text)) -(5 rows) - -SELECT count(*) FROM dupindexcols - WHERE f1 BETWEEN 'WA' AND 'ZZZ' and id < 1000 and f1 ~<~ 'YX'; - count -------- - 97 -(1 row) - --- --- Check that index scans with =ANY indexquals return rows in index order --- -explain (costs off) -SELECT unique1 FROM tenk1 -WHERE unique1 IN (1,42,7) -ORDER BY unique1; - QUERY PLAN -------------------------------------------------------- - Index Only Scan using tenk1_unique1 on tenk1 - Index Cond: (unique1 = ANY ('{1,42,7}'::integer[])) -(2 rows) - -SELECT unique1 FROM tenk1 -WHERE unique1 IN (1,42,7) -ORDER BY unique1; - unique1 ---------- - 1 - 7 - 42 -(3 rows) - --- Non-required array scan key on "tenthous": -explain (costs off) -SELECT thousand, tenthous FROM tenk1 -WHERE thousand < 2 AND tenthous IN (1001,3000) -ORDER BY thousand; - QUERY PLAN --------------------------------------------------------------------------------- - Index Only Scan using tenk1_thous_tenthous on tenk1 - Index Cond: ((thousand < 2) AND (tenthous = ANY ('{1001,3000}'::integer[]))) -(2 rows) - -SELECT thousand, tenthous FROM tenk1 -WHERE thousand < 2 AND tenthous IN (1001,3000) -ORDER BY thousand; - thousand | tenthous -----------+---------- - 0 | 3000 - 1 | 1001 -(2 rows) - --- Non-required array scan key on "tenthous", backward scan: -explain (costs off) -SELECT thousand, tenthous FROM tenk1 -WHERE thousand < 2 AND tenthous IN (1001,3000) -ORDER BY thousand DESC, tenthous DESC; - QUERY PLAN --------------------------------------------------------------------------------- - Index Only Scan Backward using tenk1_thous_tenthous on tenk1 - Index Cond: ((thousand < 2) AND (tenthous = ANY ('{1001,3000}'::integer[]))) -(2 rows) - -SELECT thousand, tenthous FROM tenk1 -WHERE thousand < 2 AND tenthous IN (1001,3000) -ORDER BY thousand DESC, tenthous DESC; - thousand | tenthous -----------+---------- - 1 | 1001 - 0 | 3000 -(2 rows) - --- --- Check elimination of redundant and contradictory index quals --- -explain (costs off) -SELECT unique1 FROM tenk1 WHERE unique1 IN (1, 42, 7) and unique1 = ANY('{7, 8, 9}'); - QUERY PLAN ----------------------------------------------------------------------------------------------------- - Index Only Scan using tenk1_unique1 on tenk1 - Index Cond: ((unique1 = ANY ('{1,42,7}'::integer[])) AND (unique1 = ANY ('{7,8,9}'::integer[]))) -(2 rows) - -SELECT unique1 FROM tenk1 WHERE unique1 IN (1, 42, 7) and unique1 = ANY('{7, 8, 9}'); - unique1 ---------- - 7 -(1 row) - -explain (costs off) -SELECT unique1 FROM tenk1 WHERE unique1 = ANY('{7, 14, 22}') and unique1 = ANY('{33, 44}'::bigint[]); - QUERY PLAN ----------------------------------------------------------------------------------------------------- - Index Only Scan using tenk1_unique1 on tenk1 - Index Cond: ((unique1 = ANY ('{7,14,22}'::integer[])) AND (unique1 = ANY ('{33,44}'::bigint[]))) -(2 rows) - -SELECT unique1 FROM tenk1 WHERE unique1 = ANY('{7, 14, 22}') and unique1 = ANY('{33, 44}'::bigint[]); - unique1 ---------- -(0 rows) - -explain (costs off) -SELECT unique1 FROM tenk1 WHERE unique1 IN (1, 42, 7) and unique1 = 1; - QUERY PLAN ---------------------------------------------------------------------------- - Index Only Scan using tenk1_unique1 on tenk1 - Index Cond: ((unique1 = ANY ('{1,42,7}'::integer[])) AND (unique1 = 1)) -(2 rows) - -SELECT unique1 FROM tenk1 WHERE unique1 IN (1, 42, 7) and unique1 = 1; - unique1 ---------- - 1 -(1 row) - -explain (costs off) -SELECT unique1 FROM tenk1 WHERE unique1 IN (1, 42, 7) and unique1 = 12345; - QUERY PLAN -------------------------------------------------------------------------------- - Index Only Scan using tenk1_unique1 on tenk1 - Index Cond: ((unique1 = ANY ('{1,42,7}'::integer[])) AND (unique1 = 12345)) -(2 rows) - -SELECT unique1 FROM tenk1 WHERE unique1 IN (1, 42, 7) and unique1 = 12345; - unique1 ---------- -(0 rows) - -explain (costs off) -SELECT unique1 FROM tenk1 WHERE unique1 IN (1, 42, 7) and unique1 >= 42; - QUERY PLAN ------------------------------------------------------------------------------ - Index Only Scan using tenk1_unique1 on tenk1 - Index Cond: ((unique1 = ANY ('{1,42,7}'::integer[])) AND (unique1 >= 42)) -(2 rows) - -SELECT unique1 FROM tenk1 WHERE unique1 IN (1, 42, 7) and unique1 >= 42; - unique1 ---------- - 42 -(1 row) - -explain (costs off) -SELECT unique1 FROM tenk1 WHERE unique1 IN (1, 42, 7) and unique1 > 42; - QUERY PLAN ----------------------------------------------------------------------------- - Index Only Scan using tenk1_unique1 on tenk1 - Index Cond: ((unique1 = ANY ('{1,42,7}'::integer[])) AND (unique1 > 42)) -(2 rows) - -SELECT unique1 FROM tenk1 WHERE unique1 IN (1, 42, 7) and unique1 > 42; - unique1 ---------- -(0 rows) - -explain (costs off) -SELECT unique1 FROM tenk1 WHERE unique1 > 9996 and unique1 >= 9999; - QUERY PLAN --------------------------------------------------------- - Index Only Scan using tenk1_unique1 on tenk1 - Index Cond: ((unique1 > 9996) AND (unique1 >= 9999)) -(2 rows) - -SELECT unique1 FROM tenk1 WHERE unique1 > 9996 and unique1 >= 9999; - unique1 ---------- - 9999 -(1 row) - -explain (costs off) -SELECT unique1 FROM tenk1 WHERE unique1 < 3 and unique1 <= 3; - QUERY PLAN --------------------------------------------------- - Index Only Scan using tenk1_unique1 on tenk1 - Index Cond: ((unique1 < 3) AND (unique1 <= 3)) -(2 rows) - -SELECT unique1 FROM tenk1 WHERE unique1 < 3 and unique1 <= 3; - unique1 ---------- - 0 - 1 - 2 -(3 rows) - -explain (costs off) -SELECT unique1 FROM tenk1 WHERE unique1 < 3 and unique1 < (-1)::bigint; - QUERY PLAN ------------------------------------------------------------- - Index Only Scan using tenk1_unique1 on tenk1 - Index Cond: ((unique1 < 3) AND (unique1 < '-1'::bigint)) -(2 rows) - -SELECT unique1 FROM tenk1 WHERE unique1 < 3 and unique1 < (-1)::bigint; - unique1 ---------- -(0 rows) - -explain (costs off) -SELECT unique1 FROM tenk1 WHERE unique1 IN (1, 42, 7) and unique1 < (-1)::bigint; - QUERY PLAN --------------------------------------------------------------------------------------- - Index Only Scan using tenk1_unique1 on tenk1 - Index Cond: ((unique1 = ANY ('{1,42,7}'::integer[])) AND (unique1 < '-1'::bigint)) -(2 rows) - -SELECT unique1 FROM tenk1 WHERE unique1 IN (1, 42, 7) and unique1 < (-1)::bigint; - unique1 ---------- -(0 rows) - --- --- Check elimination of constant-NULL subexpressions --- -explain (costs off) - select * from tenk1 where (thousand, tenthous) in ((1,1001), (null,null)); - QUERY PLAN ------------------------------------------------------- - Index Scan using tenk1_thous_tenthous on tenk1 - Index Cond: ((thousand = 1) AND (tenthous = 1001)) -(2 rows) - --- --- Check matching of boolean index columns to WHERE conditions and sort keys --- -create temp table boolindex (b bool, i int, unique(b, i), junk float); -explain (costs off) - select * from boolindex order by b, i limit 10; - QUERY PLAN -------------------------------------------------------- - Limit - -> Index Scan using boolindex_b_i_key on boolindex -(2 rows) - -explain (costs off) - select * from boolindex where b order by i limit 10; - QUERY PLAN -------------------------------------------------------- - Limit - -> Index Scan using boolindex_b_i_key on boolindex - Index Cond: (b = true) -(3 rows) - -explain (costs off) - select * from boolindex where b = true order by i desc limit 10; - QUERY PLAN ----------------------------------------------------------------- - Limit - -> Index Scan Backward using boolindex_b_i_key on boolindex - Index Cond: (b = true) -(3 rows) - -explain (costs off) - select * from boolindex where not b order by i limit 10; - QUERY PLAN -------------------------------------------------------- - Limit - -> Index Scan using boolindex_b_i_key on boolindex - Index Cond: (b = false) -(3 rows) - -explain (costs off) - select * from boolindex where b is true order by i desc limit 10; - QUERY PLAN ----------------------------------------------------------------- - Limit - -> Index Scan Backward using boolindex_b_i_key on boolindex - Index Cond: (b = true) -(3 rows) - -explain (costs off) - select * from boolindex where b is false order by i desc limit 10; - QUERY PLAN ----------------------------------------------------------------- - Limit - -> Index Scan Backward using boolindex_b_i_key on boolindex - Index Cond: (b = false) -(3 rows) - --- --- REINDEX (VERBOSE) --- -CREATE TABLE reindex_verbose(id integer primary key); -\set VERBOSITY terse \\ -- suppress machine-dependent details -REINDEX (VERBOSE) TABLE reindex_verbose; -INFO: index "reindex_verbose_pkey" was reindexed -\set VERBOSITY default -DROP TABLE reindex_verbose; --- --- REINDEX CONCURRENTLY --- -CREATE TABLE concur_reindex_tab (c1 int); --- REINDEX -REINDEX TABLE concur_reindex_tab; -- notice -NOTICE: table "concur_reindex_tab" has no indexes to reindex -REINDEX (CONCURRENTLY) TABLE concur_reindex_tab; -- notice -NOTICE: table "concur_reindex_tab" has no indexes that can be reindexed concurrently -ALTER TABLE concur_reindex_tab ADD COLUMN c2 text; -- add toast index --- Normal index with integer column -CREATE UNIQUE INDEX concur_reindex_ind1 ON concur_reindex_tab(c1); --- Normal index with text column -CREATE INDEX concur_reindex_ind2 ON concur_reindex_tab(c2); --- UNIQUE index with expression -CREATE UNIQUE INDEX concur_reindex_ind3 ON concur_reindex_tab(abs(c1)); --- Duplicate column names -CREATE INDEX concur_reindex_ind4 ON concur_reindex_tab(c1, c1, c2); --- Create table for check on foreign key dependence switch with indexes swapped -ALTER TABLE concur_reindex_tab ADD PRIMARY KEY USING INDEX concur_reindex_ind1; -CREATE TABLE concur_reindex_tab2 (c1 int REFERENCES concur_reindex_tab); -INSERT INTO concur_reindex_tab VALUES (1, 'a'); -INSERT INTO concur_reindex_tab VALUES (2, 'a'); --- Reindex concurrently of exclusion constraint currently not supported -CREATE TABLE concur_reindex_tab3 (c1 int, c2 int4range, EXCLUDE USING gist (c2 WITH &&)); -INSERT INTO concur_reindex_tab3 VALUES (3, '[1,2]'); -REINDEX INDEX CONCURRENTLY concur_reindex_tab3_c2_excl; -- error -ERROR: concurrent index creation for exclusion constraints is not supported -REINDEX TABLE CONCURRENTLY concur_reindex_tab3; -- succeeds with warning -WARNING: cannot reindex exclusion constraint index "public.concur_reindex_tab3_c2_excl" concurrently, skipping -INSERT INTO concur_reindex_tab3 VALUES (4, '[2,4]'); -ERROR: conflicting key value violates exclusion constraint "concur_reindex_tab3_c2_excl" -DETAIL: Key (c2)=([2,5)) conflicts with existing key (c2)=([1,3)). --- Check materialized views -CREATE MATERIALIZED VIEW concur_reindex_matview AS SELECT * FROM concur_reindex_tab; --- Dependency lookup before and after the follow-up REINDEX commands. --- These should remain consistent. -SELECT pg_describe_object(classid, objid, objsubid) as obj, - pg_describe_object(refclassid,refobjid,refobjsubid) as objref, - deptype -FROM pg_depend -WHERE classid = 'pg_class'::regclass AND - objid in ('concur_reindex_tab'::regclass, - 'concur_reindex_ind1'::regclass, - 'concur_reindex_ind2'::regclass, - 'concur_reindex_ind3'::regclass, - 'concur_reindex_ind4'::regclass, - 'concur_reindex_matview'::regclass) - ORDER BY 1, 2; - obj | objref | deptype -------------------------------------------+------------------------------------------------------------+--------- - index concur_reindex_ind1 | constraint concur_reindex_ind1 on table concur_reindex_tab | i - index concur_reindex_ind2 | column c2 of table concur_reindex_tab | a - index concur_reindex_ind3 | column c1 of table concur_reindex_tab | a - index concur_reindex_ind3 | table concur_reindex_tab | a - index concur_reindex_ind4 | column c1 of table concur_reindex_tab | a - index concur_reindex_ind4 | column c2 of table concur_reindex_tab | a - materialized view concur_reindex_matview | access method tde_heap | n - materialized view concur_reindex_matview | schema public | n - table concur_reindex_tab | access method tde_heap | n - table concur_reindex_tab | schema public | n -(10 rows) - -REINDEX INDEX CONCURRENTLY concur_reindex_ind1; -REINDEX TABLE CONCURRENTLY concur_reindex_tab; -REINDEX TABLE CONCURRENTLY concur_reindex_matview; -SELECT pg_describe_object(classid, objid, objsubid) as obj, - pg_describe_object(refclassid,refobjid,refobjsubid) as objref, - deptype -FROM pg_depend -WHERE classid = 'pg_class'::regclass AND - objid in ('concur_reindex_tab'::regclass, - 'concur_reindex_ind1'::regclass, - 'concur_reindex_ind2'::regclass, - 'concur_reindex_ind3'::regclass, - 'concur_reindex_ind4'::regclass, - 'concur_reindex_matview'::regclass) - ORDER BY 1, 2; - obj | objref | deptype -------------------------------------------+------------------------------------------------------------+--------- - index concur_reindex_ind1 | constraint concur_reindex_ind1 on table concur_reindex_tab | i - index concur_reindex_ind2 | column c2 of table concur_reindex_tab | a - index concur_reindex_ind3 | column c1 of table concur_reindex_tab | a - index concur_reindex_ind3 | table concur_reindex_tab | a - index concur_reindex_ind4 | column c1 of table concur_reindex_tab | a - index concur_reindex_ind4 | column c2 of table concur_reindex_tab | a - materialized view concur_reindex_matview | access method tde_heap | n - materialized view concur_reindex_matview | schema public | n - table concur_reindex_tab | access method tde_heap | n - table concur_reindex_tab | schema public | n -(10 rows) - --- Check that comments are preserved -CREATE TABLE testcomment (i int); -CREATE INDEX testcomment_idx1 ON testcomment (i); -COMMENT ON INDEX testcomment_idx1 IS 'test comment'; -SELECT obj_description('testcomment_idx1'::regclass, 'pg_class'); - obj_description ------------------ - test comment -(1 row) - -REINDEX TABLE testcomment; -SELECT obj_description('testcomment_idx1'::regclass, 'pg_class'); - obj_description ------------------ - test comment -(1 row) - -REINDEX TABLE CONCURRENTLY testcomment ; -SELECT obj_description('testcomment_idx1'::regclass, 'pg_class'); - obj_description ------------------ - test comment -(1 row) - -DROP TABLE testcomment; --- Check that indisclustered updates are preserved -CREATE TABLE concur_clustered(i int); -CREATE INDEX concur_clustered_i_idx ON concur_clustered(i); -ALTER TABLE concur_clustered CLUSTER ON concur_clustered_i_idx; -REINDEX TABLE CONCURRENTLY concur_clustered; -SELECT indexrelid::regclass, indisclustered FROM pg_index - WHERE indrelid = 'concur_clustered'::regclass; - indexrelid | indisclustered -------------------------+---------------- - concur_clustered_i_idx | t -(1 row) - -DROP TABLE concur_clustered; --- Check that indisreplident updates are preserved. -CREATE TABLE concur_replident(i int NOT NULL); -CREATE UNIQUE INDEX concur_replident_i_idx ON concur_replident(i); -ALTER TABLE concur_replident REPLICA IDENTITY - USING INDEX concur_replident_i_idx; -SELECT indexrelid::regclass, indisreplident FROM pg_index - WHERE indrelid = 'concur_replident'::regclass; - indexrelid | indisreplident -------------------------+---------------- - concur_replident_i_idx | t -(1 row) - -REINDEX TABLE CONCURRENTLY concur_replident; -SELECT indexrelid::regclass, indisreplident FROM pg_index - WHERE indrelid = 'concur_replident'::regclass; - indexrelid | indisreplident -------------------------+---------------- - concur_replident_i_idx | t -(1 row) - -DROP TABLE concur_replident; --- Check that opclass parameters are preserved -CREATE TABLE concur_appclass_tab(i tsvector, j tsvector, k tsvector); -CREATE INDEX concur_appclass_ind on concur_appclass_tab - USING gist (i tsvector_ops (siglen='1000'), j tsvector_ops (siglen='500')); -CREATE INDEX concur_appclass_ind_2 on concur_appclass_tab - USING gist (k tsvector_ops (siglen='300'), j tsvector_ops); -REINDEX TABLE CONCURRENTLY concur_appclass_tab; -\d concur_appclass_tab - Table "public.concur_appclass_tab" - Column | Type | Collation | Nullable | Default ---------+----------+-----------+----------+--------- - i | tsvector | | | - j | tsvector | | | - k | tsvector | | | -Indexes: - "concur_appclass_ind" gist (i tsvector_ops (siglen='1000'), j tsvector_ops (siglen='500')) - "concur_appclass_ind_2" gist (k tsvector_ops (siglen='300'), j) - -DROP TABLE concur_appclass_tab; --- Partitions --- Create some partitioned tables -CREATE TABLE concur_reindex_part (c1 int, c2 int) PARTITION BY RANGE (c1); -CREATE TABLE concur_reindex_part_0 PARTITION OF concur_reindex_part - FOR VALUES FROM (0) TO (10) PARTITION BY list (c2); -CREATE TABLE concur_reindex_part_0_1 PARTITION OF concur_reindex_part_0 - FOR VALUES IN (1); -CREATE TABLE concur_reindex_part_0_2 PARTITION OF concur_reindex_part_0 - FOR VALUES IN (2); --- This partitioned table will have no partitions. -CREATE TABLE concur_reindex_part_10 PARTITION OF concur_reindex_part - FOR VALUES FROM (10) TO (20) PARTITION BY list (c2); --- Create some partitioned indexes -CREATE INDEX concur_reindex_part_index ON ONLY concur_reindex_part (c1); -CREATE INDEX concur_reindex_part_index_0 ON ONLY concur_reindex_part_0 (c1); -ALTER INDEX concur_reindex_part_index ATTACH PARTITION concur_reindex_part_index_0; --- This partitioned index will have no partitions. -CREATE INDEX concur_reindex_part_index_10 ON ONLY concur_reindex_part_10 (c1); -ALTER INDEX concur_reindex_part_index ATTACH PARTITION concur_reindex_part_index_10; -CREATE INDEX concur_reindex_part_index_0_1 ON ONLY concur_reindex_part_0_1 (c1); -ALTER INDEX concur_reindex_part_index_0 ATTACH PARTITION concur_reindex_part_index_0_1; -CREATE INDEX concur_reindex_part_index_0_2 ON ONLY concur_reindex_part_0_2 (c1); -ALTER INDEX concur_reindex_part_index_0 ATTACH PARTITION concur_reindex_part_index_0_2; -SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_index') - ORDER BY relid, level; - relid | parentrelid | level --------------------------------+-----------------------------+------- - concur_reindex_part_index | | 0 - concur_reindex_part_index_0 | concur_reindex_part_index | 1 - concur_reindex_part_index_10 | concur_reindex_part_index | 1 - concur_reindex_part_index_0_1 | concur_reindex_part_index_0 | 2 - concur_reindex_part_index_0_2 | concur_reindex_part_index_0 | 2 -(5 rows) - -SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_index') - ORDER BY relid, level; - relid | parentrelid | level --------------------------------+-----------------------------+------- - concur_reindex_part_index | | 0 - concur_reindex_part_index_0 | concur_reindex_part_index | 1 - concur_reindex_part_index_10 | concur_reindex_part_index | 1 - concur_reindex_part_index_0_1 | concur_reindex_part_index_0 | 2 - concur_reindex_part_index_0_2 | concur_reindex_part_index_0 | 2 -(5 rows) - --- REINDEX should preserve dependencies of partition tree. -SELECT pg_describe_object(classid, objid, objsubid) as obj, - pg_describe_object(refclassid,refobjid,refobjsubid) as objref, - deptype -FROM pg_depend -WHERE classid = 'pg_class'::regclass AND - objid in ('concur_reindex_part'::regclass, - 'concur_reindex_part_0'::regclass, - 'concur_reindex_part_0_1'::regclass, - 'concur_reindex_part_0_2'::regclass, - 'concur_reindex_part_index'::regclass, - 'concur_reindex_part_index_0'::regclass, - 'concur_reindex_part_index_0_1'::regclass, - 'concur_reindex_part_index_0_2'::regclass) - ORDER BY 1, 2; - obj | objref | deptype -------------------------------------------+--------------------------------------------+--------- - column c1 of table concur_reindex_part | table concur_reindex_part | i - column c2 of table concur_reindex_part_0 | table concur_reindex_part_0 | i - index concur_reindex_part_index | column c1 of table concur_reindex_part | a - index concur_reindex_part_index_0 | column c1 of table concur_reindex_part_0 | a - index concur_reindex_part_index_0 | index concur_reindex_part_index | P - index concur_reindex_part_index_0 | table concur_reindex_part_0 | S - index concur_reindex_part_index_0_1 | column c1 of table concur_reindex_part_0_1 | a - index concur_reindex_part_index_0_1 | index concur_reindex_part_index_0 | P - index concur_reindex_part_index_0_1 | table concur_reindex_part_0_1 | S - index concur_reindex_part_index_0_2 | column c1 of table concur_reindex_part_0_2 | a - index concur_reindex_part_index_0_2 | index concur_reindex_part_index_0 | P - index concur_reindex_part_index_0_2 | table concur_reindex_part_0_2 | S - table concur_reindex_part | schema public | n - table concur_reindex_part_0 | schema public | n - table concur_reindex_part_0 | table concur_reindex_part | a - table concur_reindex_part_0_1 | access method tde_heap | n - table concur_reindex_part_0_1 | schema public | n - table concur_reindex_part_0_1 | table concur_reindex_part_0 | a - table concur_reindex_part_0_2 | access method tde_heap | n - table concur_reindex_part_0_2 | schema public | n - table concur_reindex_part_0_2 | table concur_reindex_part_0 | a -(21 rows) - -REINDEX INDEX CONCURRENTLY concur_reindex_part_index_0_1; -REINDEX INDEX CONCURRENTLY concur_reindex_part_index_0_2; -SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_index') - ORDER BY relid, level; - relid | parentrelid | level --------------------------------+-----------------------------+------- - concur_reindex_part_index | | 0 - concur_reindex_part_index_0 | concur_reindex_part_index | 1 - concur_reindex_part_index_10 | concur_reindex_part_index | 1 - concur_reindex_part_index_0_1 | concur_reindex_part_index_0 | 2 - concur_reindex_part_index_0_2 | concur_reindex_part_index_0 | 2 -(5 rows) - -REINDEX TABLE CONCURRENTLY concur_reindex_part_0_1; -REINDEX TABLE CONCURRENTLY concur_reindex_part_0_2; -SELECT pg_describe_object(classid, objid, objsubid) as obj, - pg_describe_object(refclassid,refobjid,refobjsubid) as objref, - deptype -FROM pg_depend -WHERE classid = 'pg_class'::regclass AND - objid in ('concur_reindex_part'::regclass, - 'concur_reindex_part_0'::regclass, - 'concur_reindex_part_0_1'::regclass, - 'concur_reindex_part_0_2'::regclass, - 'concur_reindex_part_index'::regclass, - 'concur_reindex_part_index_0'::regclass, - 'concur_reindex_part_index_0_1'::regclass, - 'concur_reindex_part_index_0_2'::regclass) - ORDER BY 1, 2; - obj | objref | deptype -------------------------------------------+--------------------------------------------+--------- - column c1 of table concur_reindex_part | table concur_reindex_part | i - column c2 of table concur_reindex_part_0 | table concur_reindex_part_0 | i - index concur_reindex_part_index | column c1 of table concur_reindex_part | a - index concur_reindex_part_index_0 | column c1 of table concur_reindex_part_0 | a - index concur_reindex_part_index_0 | index concur_reindex_part_index | P - index concur_reindex_part_index_0 | table concur_reindex_part_0 | S - index concur_reindex_part_index_0_1 | column c1 of table concur_reindex_part_0_1 | a - index concur_reindex_part_index_0_1 | index concur_reindex_part_index_0 | P - index concur_reindex_part_index_0_1 | table concur_reindex_part_0_1 | S - index concur_reindex_part_index_0_2 | column c1 of table concur_reindex_part_0_2 | a - index concur_reindex_part_index_0_2 | index concur_reindex_part_index_0 | P - index concur_reindex_part_index_0_2 | table concur_reindex_part_0_2 | S - table concur_reindex_part | schema public | n - table concur_reindex_part_0 | schema public | n - table concur_reindex_part_0 | table concur_reindex_part | a - table concur_reindex_part_0_1 | access method tde_heap | n - table concur_reindex_part_0_1 | schema public | n - table concur_reindex_part_0_1 | table concur_reindex_part_0 | a - table concur_reindex_part_0_2 | access method tde_heap | n - table concur_reindex_part_0_2 | schema public | n - table concur_reindex_part_0_2 | table concur_reindex_part_0 | a -(21 rows) - -SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_index') - ORDER BY relid, level; - relid | parentrelid | level --------------------------------+-----------------------------+------- - concur_reindex_part_index | | 0 - concur_reindex_part_index_0 | concur_reindex_part_index | 1 - concur_reindex_part_index_10 | concur_reindex_part_index | 1 - concur_reindex_part_index_0_1 | concur_reindex_part_index_0 | 2 - concur_reindex_part_index_0_2 | concur_reindex_part_index_0 | 2 -(5 rows) - --- REINDEX for partitioned indexes --- REINDEX TABLE fails for partitioned indexes --- Top-most parent index -REINDEX TABLE concur_reindex_part_index; -- error -ERROR: "concur_reindex_part_index" is not a table or materialized view -REINDEX TABLE CONCURRENTLY concur_reindex_part_index; -- error -ERROR: "concur_reindex_part_index" is not a table or materialized view --- Partitioned index with no leaves -REINDEX TABLE concur_reindex_part_index_10; -- error -ERROR: "concur_reindex_part_index_10" is not a table or materialized view -REINDEX TABLE CONCURRENTLY concur_reindex_part_index_10; -- error -ERROR: "concur_reindex_part_index_10" is not a table or materialized view --- Cannot run in a transaction block -BEGIN; -REINDEX INDEX concur_reindex_part_index; -ERROR: REINDEX INDEX cannot run inside a transaction block -CONTEXT: while reindexing partitioned index "public.concur_reindex_part_index" -ROLLBACK; --- Helper functions to track changes of relfilenodes in a partition tree. --- Create a table tracking the relfilenode state. -CREATE OR REPLACE FUNCTION create_relfilenode_part(relname text, indname text) - RETURNS VOID AS - $func$ - BEGIN - EXECUTE format(' - CREATE TABLE %I AS - SELECT oid, relname, relfilenode, relkind, reltoastrelid - FROM pg_class - WHERE oid IN - (SELECT relid FROM pg_partition_tree(''%I''));', - relname, indname); - END - $func$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION compare_relfilenode_part(tabname text) - RETURNS TABLE (relname name, relkind "char", state text) AS - $func$ - BEGIN - RETURN QUERY EXECUTE - format( - 'SELECT b.relname, - b.relkind, - CASE WHEN a.relfilenode = b.relfilenode THEN ''relfilenode is unchanged'' - ELSE ''relfilenode has changed'' END - -- Do not join with OID here as CONCURRENTLY changes it. - FROM %I b JOIN pg_class a ON b.relname = a.relname - ORDER BY 1;', tabname); - END - $func$ LANGUAGE plpgsql; --- Check that expected relfilenodes are changed, non-concurrent case. -SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_index'); - create_relfilenode_part -------------------------- - -(1 row) - -REINDEX INDEX concur_reindex_part_index; -SELECT * FROM compare_relfilenode_part('reindex_index_status'); - relname | relkind | state --------------------------------+---------+-------------------------- - concur_reindex_part_index | I | relfilenode is unchanged - concur_reindex_part_index_0 | I | relfilenode is unchanged - concur_reindex_part_index_0_1 | i | relfilenode has changed - concur_reindex_part_index_0_2 | i | relfilenode has changed - concur_reindex_part_index_10 | I | relfilenode is unchanged -(5 rows) - -DROP TABLE reindex_index_status; --- concurrent case. -SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_index'); - create_relfilenode_part -------------------------- - -(1 row) - -REINDEX INDEX CONCURRENTLY concur_reindex_part_index; -SELECT * FROM compare_relfilenode_part('reindex_index_status'); - relname | relkind | state --------------------------------+---------+-------------------------- - concur_reindex_part_index | I | relfilenode is unchanged - concur_reindex_part_index_0 | I | relfilenode is unchanged - concur_reindex_part_index_0_1 | i | relfilenode has changed - concur_reindex_part_index_0_2 | i | relfilenode has changed - concur_reindex_part_index_10 | I | relfilenode is unchanged -(5 rows) - -DROP TABLE reindex_index_status; --- REINDEX for partitioned tables --- REINDEX INDEX fails for partitioned tables --- Top-most parent -REINDEX INDEX concur_reindex_part; -- error -ERROR: "concur_reindex_part" is not an index -REINDEX INDEX CONCURRENTLY concur_reindex_part; -- error -ERROR: "concur_reindex_part" is not an index --- Partitioned with no leaves -REINDEX INDEX concur_reindex_part_10; -- error -ERROR: "concur_reindex_part_10" is not an index -REINDEX INDEX CONCURRENTLY concur_reindex_part_10; -- error -ERROR: "concur_reindex_part_10" is not an index --- Cannot run in a transaction block -BEGIN; -REINDEX TABLE concur_reindex_part; -ERROR: REINDEX TABLE cannot run inside a transaction block -CONTEXT: while reindexing partitioned table "public.concur_reindex_part" -ROLLBACK; --- Check that expected relfilenodes are changed, non-concurrent case. --- Note that the partition tree changes of the *indexes* need to be checked. -SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_index'); - create_relfilenode_part -------------------------- - -(1 row) - -REINDEX TABLE concur_reindex_part; -SELECT * FROM compare_relfilenode_part('reindex_index_status'); - relname | relkind | state --------------------------------+---------+-------------------------- - concur_reindex_part_index | I | relfilenode is unchanged - concur_reindex_part_index_0 | I | relfilenode is unchanged - concur_reindex_part_index_0_1 | i | relfilenode has changed - concur_reindex_part_index_0_2 | i | relfilenode has changed - concur_reindex_part_index_10 | I | relfilenode is unchanged -(5 rows) - -DROP TABLE reindex_index_status; --- concurrent case. -SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_index'); - create_relfilenode_part -------------------------- - -(1 row) - -REINDEX TABLE CONCURRENTLY concur_reindex_part; -SELECT * FROM compare_relfilenode_part('reindex_index_status'); - relname | relkind | state --------------------------------+---------+-------------------------- - concur_reindex_part_index | I | relfilenode is unchanged - concur_reindex_part_index_0 | I | relfilenode is unchanged - concur_reindex_part_index_0_1 | i | relfilenode has changed - concur_reindex_part_index_0_2 | i | relfilenode has changed - concur_reindex_part_index_10 | I | relfilenode is unchanged -(5 rows) - -DROP TABLE reindex_index_status; -DROP FUNCTION create_relfilenode_part; -DROP FUNCTION compare_relfilenode_part; --- Cleanup of partition tree used for REINDEX test. -DROP TABLE concur_reindex_part; --- Check errors --- Cannot run inside a transaction block -BEGIN; -REINDEX TABLE CONCURRENTLY concur_reindex_tab; -ERROR: REINDEX CONCURRENTLY cannot run inside a transaction block -COMMIT; -REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation -ERROR: cannot reindex system catalogs concurrently -REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -ERROR: cannot reindex system catalogs concurrently --- These are the toast table and index of pg_authid. -REINDEX TABLE CONCURRENTLY pg_toast.pg_toast_1260; -- no catalog toast table -ERROR: cannot reindex system catalogs concurrently -REINDEX INDEX CONCURRENTLY pg_toast.pg_toast_1260_index; -- no catalog toast index -ERROR: cannot reindex system catalogs concurrently -REINDEX SYSTEM CONCURRENTLY postgres; -- not allowed for SYSTEM -ERROR: cannot reindex system catalogs concurrently -REINDEX (CONCURRENTLY) SYSTEM postgres; -- ditto -ERROR: cannot reindex system catalogs concurrently -REINDEX (CONCURRENTLY) SYSTEM; -- ditto -ERROR: cannot reindex system catalogs concurrently --- Warns about catalog relations -REINDEX SCHEMA CONCURRENTLY pg_catalog; -WARNING: cannot reindex system catalogs concurrently, skipping all --- Not the current database -REINDEX DATABASE not_current_database; -ERROR: can only reindex the currently open database --- Check the relation status, there should not be invalid indexes -\d concur_reindex_tab - Table "public.concur_reindex_tab" - Column | Type | Collation | Nullable | Default ---------+---------+-----------+----------+--------- - c1 | integer | | not null | - c2 | text | | | -Indexes: - "concur_reindex_ind1" PRIMARY KEY, btree (c1) - "concur_reindex_ind2" btree (c2) - "concur_reindex_ind3" UNIQUE, btree (abs(c1)) - "concur_reindex_ind4" btree (c1, c1, c2) -Referenced by: - TABLE "concur_reindex_tab2" CONSTRAINT "concur_reindex_tab2_c1_fkey" FOREIGN KEY (c1) REFERENCES concur_reindex_tab(c1) - -DROP MATERIALIZED VIEW concur_reindex_matview; -DROP TABLE concur_reindex_tab, concur_reindex_tab2, concur_reindex_tab3; --- Check handling of invalid indexes -CREATE TABLE concur_reindex_tab4 (c1 int); -INSERT INTO concur_reindex_tab4 VALUES (1), (1), (2); --- This trick creates an invalid index. -CREATE UNIQUE INDEX CONCURRENTLY concur_reindex_ind5 ON concur_reindex_tab4 (c1); -ERROR: could not create unique index "concur_reindex_ind5" -DETAIL: Key (c1)=(1) is duplicated. --- Reindexing concurrently this index fails with the same failure. --- The extra index created is itself invalid, and can be dropped. -REINDEX INDEX CONCURRENTLY concur_reindex_ind5; -ERROR: could not create unique index "concur_reindex_ind5_ccnew" -DETAIL: Key (c1)=(1) is duplicated. -\d concur_reindex_tab4 - Table "public.concur_reindex_tab4" - Column | Type | Collation | Nullable | Default ---------+---------+-----------+----------+--------- - c1 | integer | | | -Indexes: - "concur_reindex_ind5" UNIQUE, btree (c1) INVALID - "concur_reindex_ind5_ccnew" UNIQUE, btree (c1) INVALID - -DROP INDEX concur_reindex_ind5_ccnew; --- This makes the previous failure go away, so the index can become valid. -DELETE FROM concur_reindex_tab4 WHERE c1 = 1; --- The invalid index is not processed when running REINDEX TABLE. -REINDEX TABLE CONCURRENTLY concur_reindex_tab4; -WARNING: skipping reindex of invalid index "public.concur_reindex_ind5" -HINT: Use DROP INDEX or REINDEX INDEX. -NOTICE: table "concur_reindex_tab4" has no indexes that can be reindexed concurrently -\d concur_reindex_tab4 - Table "public.concur_reindex_tab4" - Column | Type | Collation | Nullable | Default ---------+---------+-----------+----------+--------- - c1 | integer | | | -Indexes: - "concur_reindex_ind5" UNIQUE, btree (c1) INVALID - --- But it is fixed with REINDEX INDEX. -REINDEX INDEX CONCURRENTLY concur_reindex_ind5; -\d concur_reindex_tab4 - Table "public.concur_reindex_tab4" - Column | Type | Collation | Nullable | Default ---------+---------+-----------+----------+--------- - c1 | integer | | | -Indexes: - "concur_reindex_ind5" UNIQUE, btree (c1) - -DROP TABLE concur_reindex_tab4; --- Check handling of indexes with expressions and predicates. The --- definitions of the rebuilt indexes should match the original --- definitions. -CREATE TABLE concur_exprs_tab (c1 int , c2 boolean); -INSERT INTO concur_exprs_tab (c1, c2) VALUES (1369652450, FALSE), - (414515746, TRUE), - (897778963, FALSE); -CREATE UNIQUE INDEX concur_exprs_index_expr - ON concur_exprs_tab ((c1::text COLLATE "C")); -CREATE UNIQUE INDEX concur_exprs_index_pred ON concur_exprs_tab (c1) - WHERE (c1::text > 500000000::text COLLATE "C"); -CREATE UNIQUE INDEX concur_exprs_index_pred_2 - ON concur_exprs_tab ((1 / c1)) - WHERE ('-H') >= (c2::TEXT) COLLATE "C"; -ALTER INDEX concur_exprs_index_expr ALTER COLUMN 1 SET STATISTICS 100; -ANALYZE concur_exprs_tab; -SELECT starelid::regclass, count(*) FROM pg_statistic WHERE starelid IN ( - 'concur_exprs_index_expr'::regclass, - 'concur_exprs_index_pred'::regclass, - 'concur_exprs_index_pred_2'::regclass) - GROUP BY starelid ORDER BY starelid::regclass::text; - starelid | count --------------------------+------- - concur_exprs_index_expr | 1 -(1 row) - -SELECT pg_get_indexdef('concur_exprs_index_expr'::regclass); - pg_get_indexdef ---------------------------------------------------------------------------------------------------------------- - CREATE UNIQUE INDEX concur_exprs_index_expr ON public.concur_exprs_tab USING btree (((c1)::text) COLLATE "C") -(1 row) - -SELECT pg_get_indexdef('concur_exprs_index_pred'::regclass); - pg_get_indexdef ----------------------------------------------------------------------------------------------------------------------------------------------- - CREATE UNIQUE INDEX concur_exprs_index_pred ON public.concur_exprs_tab USING btree (c1) WHERE ((c1)::text > ((500000000)::text COLLATE "C")) -(1 row) - -SELECT pg_get_indexdef('concur_exprs_index_pred_2'::regclass); - pg_get_indexdef --------------------------------------------------------------------------------------------------------------------------------------------------- - CREATE UNIQUE INDEX concur_exprs_index_pred_2 ON public.concur_exprs_tab USING btree (((1 / c1))) WHERE ('-H'::text >= ((c2)::text COLLATE "C")) -(1 row) - -REINDEX TABLE CONCURRENTLY concur_exprs_tab; -SELECT pg_get_indexdef('concur_exprs_index_expr'::regclass); - pg_get_indexdef ---------------------------------------------------------------------------------------------------------------- - CREATE UNIQUE INDEX concur_exprs_index_expr ON public.concur_exprs_tab USING btree (((c1)::text) COLLATE "C") -(1 row) - -SELECT pg_get_indexdef('concur_exprs_index_pred'::regclass); - pg_get_indexdef ----------------------------------------------------------------------------------------------------------------------------------------------- - CREATE UNIQUE INDEX concur_exprs_index_pred ON public.concur_exprs_tab USING btree (c1) WHERE ((c1)::text > ((500000000)::text COLLATE "C")) -(1 row) - -SELECT pg_get_indexdef('concur_exprs_index_pred_2'::regclass); - pg_get_indexdef --------------------------------------------------------------------------------------------------------------------------------------------------- - CREATE UNIQUE INDEX concur_exprs_index_pred_2 ON public.concur_exprs_tab USING btree (((1 / c1))) WHERE ('-H'::text >= ((c2)::text COLLATE "C")) -(1 row) - --- ALTER TABLE recreates the indexes, which should keep their collations. -ALTER TABLE concur_exprs_tab ALTER c2 TYPE TEXT; -SELECT pg_get_indexdef('concur_exprs_index_expr'::regclass); - pg_get_indexdef ---------------------------------------------------------------------------------------------------------------- - CREATE UNIQUE INDEX concur_exprs_index_expr ON public.concur_exprs_tab USING btree (((c1)::text) COLLATE "C") -(1 row) - -SELECT pg_get_indexdef('concur_exprs_index_pred'::regclass); - pg_get_indexdef ----------------------------------------------------------------------------------------------------------------------------------------------- - CREATE UNIQUE INDEX concur_exprs_index_pred ON public.concur_exprs_tab USING btree (c1) WHERE ((c1)::text > ((500000000)::text COLLATE "C")) -(1 row) - -SELECT pg_get_indexdef('concur_exprs_index_pred_2'::regclass); - pg_get_indexdef ------------------------------------------------------------------------------------------------------------------------------------------- - CREATE UNIQUE INDEX concur_exprs_index_pred_2 ON public.concur_exprs_tab USING btree (((1 / c1))) WHERE ('-H'::text >= (c2 COLLATE "C")) -(1 row) - --- Statistics should remain intact. -SELECT starelid::regclass, count(*) FROM pg_statistic WHERE starelid IN ( - 'concur_exprs_index_expr'::regclass, - 'concur_exprs_index_pred'::regclass, - 'concur_exprs_index_pred_2'::regclass) - GROUP BY starelid ORDER BY starelid::regclass::text; - starelid | count --------------------------+------- - concur_exprs_index_expr | 1 -(1 row) - --- attstattarget should remain intact -SELECT attrelid::regclass, attnum, attstattarget - FROM pg_attribute WHERE attrelid IN ( - 'concur_exprs_index_expr'::regclass, - 'concur_exprs_index_pred'::regclass, - 'concur_exprs_index_pred_2'::regclass) - ORDER BY attrelid::regclass::text, attnum; - attrelid | attnum | attstattarget ----------------------------+--------+--------------- - concur_exprs_index_expr | 1 | 100 - concur_exprs_index_pred | 1 | - concur_exprs_index_pred_2 | 1 | -(3 rows) - -DROP TABLE concur_exprs_tab; --- Temporary tables and on-commit actions, where CONCURRENTLY is ignored. --- ON COMMIT PRESERVE ROWS, the default. -CREATE TEMP TABLE concur_temp_tab_1 (c1 int, c2 text) - ON COMMIT PRESERVE ROWS; -INSERT INTO concur_temp_tab_1 VALUES (1, 'foo'), (2, 'bar'); -CREATE INDEX concur_temp_ind_1 ON concur_temp_tab_1(c2); -REINDEX TABLE CONCURRENTLY concur_temp_tab_1; -REINDEX INDEX CONCURRENTLY concur_temp_ind_1; --- Still fails in transaction blocks -BEGIN; -REINDEX INDEX CONCURRENTLY concur_temp_ind_1; -ERROR: REINDEX CONCURRENTLY cannot run inside a transaction block -COMMIT; --- ON COMMIT DELETE ROWS -CREATE TEMP TABLE concur_temp_tab_2 (c1 int, c2 text) - ON COMMIT DELETE ROWS; -CREATE INDEX concur_temp_ind_2 ON concur_temp_tab_2(c2); -REINDEX TABLE CONCURRENTLY concur_temp_tab_2; -REINDEX INDEX CONCURRENTLY concur_temp_ind_2; --- ON COMMIT DROP -BEGIN; -CREATE TEMP TABLE concur_temp_tab_3 (c1 int, c2 text) - ON COMMIT PRESERVE ROWS; -INSERT INTO concur_temp_tab_3 VALUES (1, 'foo'), (2, 'bar'); -CREATE INDEX concur_temp_ind_3 ON concur_temp_tab_3(c2); --- Fails when running in a transaction -REINDEX INDEX CONCURRENTLY concur_temp_ind_3; -ERROR: REINDEX CONCURRENTLY cannot run inside a transaction block -COMMIT; --- REINDEX SCHEMA processes all temporary relations -CREATE TABLE reindex_temp_before AS -SELECT oid, relname, relfilenode, relkind, reltoastrelid - FROM pg_class - WHERE relname IN ('concur_temp_ind_1', 'concur_temp_ind_2'); -SELECT pg_my_temp_schema()::regnamespace as temp_schema_name \gset -REINDEX SCHEMA CONCURRENTLY :temp_schema_name; -SELECT b.relname, - b.relkind, - CASE WHEN a.relfilenode = b.relfilenode THEN 'relfilenode is unchanged' - ELSE 'relfilenode has changed' END - FROM reindex_temp_before b JOIN pg_class a ON b.oid = a.oid - ORDER BY 1; - relname | relkind | case --------------------+---------+------------------------- - concur_temp_ind_1 | i | relfilenode has changed - concur_temp_ind_2 | i | relfilenode has changed -(2 rows) - -DROP TABLE concur_temp_tab_1, concur_temp_tab_2, reindex_temp_before; --- --- REINDEX SCHEMA --- -REINDEX SCHEMA schema_to_reindex; -- failure, schema does not exist -ERROR: schema "schema_to_reindex" does not exist -CREATE SCHEMA schema_to_reindex; -SET search_path = 'schema_to_reindex'; -CREATE TABLE table1(col1 SERIAL PRIMARY KEY); -INSERT INTO table1 SELECT generate_series(1,400); -CREATE TABLE table2(col1 SERIAL PRIMARY KEY, col2 TEXT NOT NULL); -INSERT INTO table2 SELECT generate_series(1,400), 'abc'; -CREATE INDEX ON table2(col2); -CREATE MATERIALIZED VIEW matview AS SELECT col1 FROM table2; -CREATE INDEX ON matview(col1); -CREATE VIEW view AS SELECT col2 FROM table2; -CREATE TABLE reindex_before AS -SELECT oid, relname, relfilenode, relkind, reltoastrelid - FROM pg_class - where relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'schema_to_reindex'); -INSERT INTO reindex_before -SELECT oid, 'pg_toast_TABLE', relfilenode, relkind, reltoastrelid -FROM pg_class WHERE oid IN - (SELECT reltoastrelid FROM reindex_before WHERE reltoastrelid > 0); -INSERT INTO reindex_before -SELECT oid, 'pg_toast_TABLE_index', relfilenode, relkind, reltoastrelid -FROM pg_class where oid in - (select indexrelid from pg_index where indrelid in - (select reltoastrelid from reindex_before where reltoastrelid > 0)); -REINDEX SCHEMA schema_to_reindex; -CREATE TABLE reindex_after AS SELECT oid, relname, relfilenode, relkind - FROM pg_class - where relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'schema_to_reindex'); -SELECT b.relname, - b.relkind, - CASE WHEN a.relfilenode = b.relfilenode THEN 'relfilenode is unchanged' - ELSE 'relfilenode has changed' END - FROM reindex_before b JOIN pg_class a ON b.oid = a.oid - ORDER BY 1; - relname | relkind | case -----------------------+---------+-------------------------- - matview | m | relfilenode is unchanged - matview_col1_idx | i | relfilenode has changed - pg_toast_TABLE | t | relfilenode is unchanged - pg_toast_TABLE_index | i | relfilenode has changed - table1 | r | relfilenode is unchanged - table1_col1_seq | S | relfilenode is unchanged - table1_pkey | i | relfilenode has changed - table2 | r | relfilenode is unchanged - table2_col1_seq | S | relfilenode is unchanged - table2_col2_idx | i | relfilenode has changed - table2_pkey | i | relfilenode has changed - view | v | relfilenode is unchanged -(12 rows) - -REINDEX SCHEMA schema_to_reindex; -BEGIN; -REINDEX SCHEMA schema_to_reindex; -- failure, cannot run in a transaction -ERROR: REINDEX SCHEMA cannot run inside a transaction block -END; --- concurrently -REINDEX SCHEMA CONCURRENTLY schema_to_reindex; --- Failure for unauthorized user -CREATE ROLE regress_reindexuser NOLOGIN; -SET SESSION ROLE regress_reindexuser; -REINDEX SCHEMA schema_to_reindex; -ERROR: must be owner of schema schema_to_reindex --- Permission failures with toast tables and indexes (pg_authid here) -RESET ROLE; -GRANT USAGE ON SCHEMA pg_toast TO regress_reindexuser; -SET SESSION ROLE regress_reindexuser; -REINDEX TABLE pg_toast.pg_toast_1260; -ERROR: permission denied for table pg_toast_1260 -REINDEX INDEX pg_toast.pg_toast_1260_index; -ERROR: permission denied for index pg_toast_1260_index --- Clean up -RESET ROLE; -REVOKE USAGE ON SCHEMA pg_toast FROM regress_reindexuser; -DROP ROLE regress_reindexuser; -DROP SCHEMA schema_to_reindex CASCADE; -NOTICE: drop cascades to 6 other objects -DETAIL: drop cascades to table table1 -drop cascades to table table2 -drop cascades to materialized view matview -drop cascades to view view -drop cascades to table reindex_before -drop cascades to table reindex_after diff --git a/src/test/regress/expected/event_trigger_1.out b/src/test/regress/expected/event_trigger_1.out index 9786bc0a00ac4..fdf2ca0124546 100644 --- a/src/test/regress/expected/event_trigger_1.out +++ b/src/test/regress/expected/event_trigger_1.out @@ -708,13 +708,13 @@ SELECT LATERAL pg_identify_object_as_address('pg_event_trigger'::regclass, e.oid, 0) as b, LATERAL pg_get_object_address(b.type, b.object_names, b.object_args) as a ORDER BY e.evtname; - evtname | descr | type | object_names | object_args | ident --------------------------------+---------------------------------------------+---------------+---------------------------------+-------------+-------------------------------------------------------------------------------- - end_rls_command | event trigger end_rls_command | event trigger | {end_rls_command} | {} | ("event trigger",,end_rls_command,end_rls_command) - pg_tde_trigger_create_index | event trigger pg_tde_trigger_create_index | event trigger | {pg_tde_trigger_create_index} | {} | ("event trigger",,pg_tde_trigger_create_index,pg_tde_trigger_create_index) - pg_tde_trigger_create_index_2 | event trigger pg_tde_trigger_create_index_2 | event trigger | {pg_tde_trigger_create_index_2} | {} | ("event trigger",,pg_tde_trigger_create_index_2,pg_tde_trigger_create_index_2) - sql_drop_command | event trigger sql_drop_command | event trigger | {sql_drop_command} | {} | ("event trigger",,sql_drop_command,sql_drop_command) - start_rls_command | event trigger start_rls_command | event trigger | {start_rls_command} | {} | ("event trigger",,start_rls_command,start_rls_command) + evtname | descr | type | object_names | object_args | ident +-------------------+---------------------------------+---------------+---------------------+-------------+-------------------------------------------------------- + end_rls_command | event trigger end_rls_command | event trigger | {end_rls_command} | {} | ("event trigger",,end_rls_command,end_rls_command) + pg_tde_ddl_end | event trigger pg_tde_ddl_end | event trigger | {pg_tde_ddl_end} | {} | ("event trigger",,pg_tde_ddl_end,pg_tde_ddl_end) + pg_tde_ddl_start | event trigger pg_tde_ddl_start | event trigger | {pg_tde_ddl_start} | {} | ("event trigger",,pg_tde_ddl_start,pg_tde_ddl_start) + sql_drop_command | event trigger sql_drop_command | event trigger | {sql_drop_command} | {} | ("event trigger",,sql_drop_command,sql_drop_command) + start_rls_command | event trigger start_rls_command | event trigger | {start_rls_command} | {} | ("event trigger",,start_rls_command,start_rls_command) (5 rows) DROP EVENT TRIGGER start_rls_command; diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c index 4cc0c35418fc7..62c5b84a7f31b 100644 --- a/src/test/regress/pg_regress.c +++ b/src/test/regress/pg_regress.c @@ -1997,8 +1997,8 @@ create_database(const char *dbname) sql_file = fopen(exec_sql, "r"); if (sql_file == NULL) { - bail("could not open \"%s\" to read extra setup file: %s", - exec_sql, strerror(errno)); + bail("could not open \"%s\" to read extra setup file: %m", + exec_sql); } while (fgets(line_buf, sizeof(line_buf), sql_file) != NULL) psql_command(dbname, "%s", line_buf); diff --git a/src/tools/pgindent/exclude_file_patterns b/src/tools/pgindent/exclude_file_patterns index 0746df4737200..76fd671dc95eb 100644 --- a/src/tools/pgindent/exclude_file_patterns +++ b/src/tools/pgindent/exclude_file_patterns @@ -64,7 +64,4 @@ src/tools/pg_bsd_indent/.* # ... and for paranoia's sake, don't touch git stuff. /\.git/ # Percona excludes -contrib/pg_tde/src16/.* -contrib/pg_tde/src17.* contrib/pg_tde/src/libkmip/.* -src/backend/nodes/nodetags.h diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 72e6d3a9865ab..2114d560ceda0 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1607,6 +1607,7 @@ ManyTestResourceKind Material MaterialPath MaterialState +MdSMgrRelationData MdfdVec Memoize MemoizeEntry @@ -3206,6 +3207,7 @@ XLogRecordBuffer XLogRecoveryCtlData XLogRedoAction XLogSegNo +XLogSmgr XLogSource XLogStats XLogwrtResult