pg_tde test PR

.git-blame-ignore-revs

@@ -14,6 +14,9 @@
 #
 # $ git log --pretty=format:"%H # %cd%n# %s" $PGINDENTGITHASH -1 --date=iso
 
+71da1f03f2f0ff18ed11e4ba6b07b6bd56705a5d # 2025-04-22 11:40:24 +0200
+# Run pgperltidy
+
 c739ae9e288c095cfe1b91ce27a2f2c075ed5da4 # 2024-08-26 16:16:09 -0700
 # Fix identation.
 

.github/CODEOWNERS

@@ -2,5 +2,5 @@
 # Order is important; the last matching pattern takes the most precedence.
 
 *                                 @dutow @dAdAbird
-/contrib/pg_tde/documentation/    @nastena1606
+/contrib/pg_tde/documentation/    @nastena1606 @Andriciuc
 /.github/                         @artemgavrilov

.github/codecov.yml

@@ -0,0 +1,45 @@
+codecov:
+  strict_yaml_branch: TDE_REL_17_STABLE
+fixes:
+  - "src/::contrib/pg_tde/src/"  # move path for codecov file mappings e.g., "src/" => "contrib/pg_tde/src/"
+coverage:
+  status:
+    project:
+      default:
+        target: 90%
+        threshold: 1%
+        base: auto
+comment:
+  layout: "header, diff, components"
+component_management:
+  default_rules:
+    statuses:
+      - type: project
+        target: auto
+        branches:
+          - "TDE_REL_17_STABLE"
+  individual_components:
+    - component_id: access
+      paths:
+        - contrib/pg_tde/src/access/**
+    - component_id: catalog
+      paths:
+        - contrib/pg_tde/src/catalog/**
+    - component_id: common
+      paths:
+        - contrib/pg_tde/src/common/**
+    - component_id: encryption
+      paths:
+        - contrib/pg_tde/src/encryption/**
+    - component_id: keyring
+      paths:
+        - contrib/pg_tde/src/keyring/**
+    - component_id: src
+      paths:
+        - contrib/pg_tde/src/*.c
+    - component_id: smgr
+      paths:
+        - contrib/pg_tde/src/smgr/**
+    - component_id: transam
+      paths:
+        - contrib/pg_tde/src/transam/**

.github/workflows/coverage.yml

@@ -0,0 +1,56 @@
+name: Code coverage
+on:
+  pull_request:
+  push:
+    branches:
+      - TDE_REL_17_STABLE
+
+jobs:
+  collect:
+    name: Collect and upload
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Install dependencies
+        run: ci_scripts/ubuntu-deps.sh
+
+      - name: Build postgres
+        run: ci_scripts/make-build.sh debug --enable-coverage
+
+      - name: Setup kmip and vault
+        run: ci_scripts/setup-keyring-servers.sh
+
+      - name: Test postgres with TDE to generate coverage
+        run: ci_scripts/make-test.sh --tde-only
+
+      - name: Collect coverage data
+        run: find . -type f -name "*.c" ! -path '*libkmip*' | xargs -t gcov -abcfu
+        working-directory: contrib/pg_tde
+
+      - name: Upload coverage data to codecov.io
+        uses: codecov/codecov-action@v5
+        with:
+          verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
+          working-directory: contrib/pg_tde
+          files: "*.c.gcov"
+
+      - name: Report on test fail
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() }}
+        with:
+          name: coverage-testlog-tde
+          path: |
+            build/testrun/
+            contrib/pg_tde/t/
+            contrib/pg_tde/results
+            contrib/pg_tde/regression.diffs
+            contrib/pg_tde/regression.out
+            contrib/pg_tde/*.gcov
+          retention-days: 3
+
+

.github/workflows/pgindent.yml

@@ -10,7 +10,7 @@ defaults:
 jobs:
   check:
     name: Check
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Clone repository
         uses: actions/checkout@v4
@@ -26,6 +26,9 @@ jobs:
 
       - name: Update typedefs
         run: ci_scripts/dump-typedefs.sh
-    
+
       - name: Run pgindent
         run: ci_scripts/run-pgindent.sh --check --diff
+
+      - name: Run pgperltidy
+        run: ci_scripts/run-pgperltidy.sh --assert-tidy --standard-error-output

.github/workflows/psp-matrix.yml

@@ -1,6 +1,9 @@
 name: PSP
 on:
   pull_request:
+  push:
+    branches:
+      - TDE_REL_17_STABLE
   workflow_dispatch:
 
 jobs:

.github/workflows/psp-reusable.yml

@@ -33,7 +33,7 @@ jobs:
       - name: Build postgres
         run: src/ci_scripts/${{ inputs.build_script }}-build.sh ${{ inputs.build_type }}
 
-      - name: 'Tar files'
+      - name: Archive pginst to artifact tar file
         run: tar -czf artifacts.tar src pginst
 
       - name: Upload build artifacts
@@ -57,7 +57,7 @@ jobs:
           name: ${{ env.artifact_name }}
           path: .
 
-      - name: 'Untar files'
+      - name: Extract artifact file
         run: tar -xzf artifacts.tar
 
       - name: Install dependencies
@@ -73,13 +73,20 @@ jobs:
         uses: actions/upload-artifact@v4
         if: ${{ failure() }}
         with:
-          name: testlog-ubuntu-${{ inputs.ubuntu_version }}.04-meson-${{ inputs.build_type }}
+          name: log-test-${{ inputs.os }}-${{ inputs.build_script }}-${{ inputs.build_type }}
           path: |
             src/build/testrun/
-            src/contrib/*/t/
-            src/contrib/*/results
+            src/contrib/*/log
             src/contrib/*/regression.diffs
             src/contrib/*/regression.out
+            src/contrib/*/results
+            src/contrib/*/tmp_check
+            src/contrib/*/t/results
+            src/src/test/*/log
+            src/src/test/*/regression.diffs
+            src/src/test/*/regression.out
+            src/src/test/*/results
+            src/src/test/*/tmp_check
           retention-days: 3
 
   test_tde:
@@ -95,27 +102,34 @@ jobs:
           name: ${{ env.artifact_name }}
           path: .
 
-      - name: 'Untar files'
+      - name: Extract artifact file
         run: tar -xzf artifacts.tar
 
       - name: Install dependencies
-        run: src/ci_scripts/ubuntu-deps.sh 
-  
+        run: src/ci_scripts/ubuntu-deps.sh
+
       - name: Setup kmip and vault
         run: src/ci_scripts/setup-keyring-servers.sh
-  
-      - name: Test postgres with TDE
-        run: src/ci_scripts/${{ inputs.build_script }}-test-tde.sh --continue
+
+      - name: Test postgres with TDE as default access method
+        run: src/ci_scripts/${{ inputs.build_script }}-test-global-tde.sh --continue
 
       - name: Report on test fail
         uses: actions/upload-artifact@v4
         if: ${{ failure() }}
         with:
-          name: testlog-tde-${{ inputs.os }}-${{  inputs.build_script }}-${{ inputs.build_type }}
+          name: log-test-global-tde-${{ inputs.os }}-${{ inputs.build_script }}-${{ inputs.build_type }}
           path: |
             src/build/testrun/
-            src/contrib/*/t/
-            src/contrib/*/results
+            src/contrib/*/log
             src/contrib/*/regression.diffs
             src/contrib/*/regression.out
+            src/contrib/*/results
+            src/contrib/*/tmp_check
+            src/contrib/*/t/results
+            src/src/test/*/log
+            src/src/test/*/regression.diffs
+            src/src/test/*/regression.out
+            src/src/test/*/results
+            src/src/test/*/tmp_check
           retention-days: 3

README.md

@@ -1,3 +1,5 @@
+### Code coverage for [pg_tde](https://github.com/percona/postgres/tree/TDE_REL_17_STABLE/contrib/pg_tde): [![codecov](https://codecov.io/github/percona/postgres/graph/badge.svg?token=Wow78BMYdP)](https://codecov.io/github/percona/postgres)
+
 Percona Server for PostgreSQL
 =============================
 

ci_scripts/backup/pg_basebackup_test.sh

@@ -104,8 +104,8 @@ setup_tde_heap(){
     sudo -u "$PG_USER" psql -p $PG_PORT -c "DROP DATABASE IF EXISTS $DB_NAME;"
     sudo -u "$PG_USER" psql -p $PG_PORT -c "CREATE DATABASE $DB_NAME;"
     sudo -u "$PG_USER" psql -d "$DB_NAME" -p "$PG_PORT" -c "CREATE EXTENSION IF NOT EXISTS pg_tde;"
-    sudo -u "$PG_USER" psql -d "$DB_NAME" -p "$PG_PORT" -c "SELECT pg_tde_add_key_provider_file('file-vault','$KEYLOCATION');"
-    sudo -u "$PG_USER" psql -d "$DB_NAME" -p "$PG_PORT" -c "SELECT pg_tde_set_principal_key('test-db-master-key','file-vault');"
+    sudo -u "$PG_USER" psql -d "$DB_NAME" -p "$PG_PORT" -c "SELECT pg_tde_add_database_key_provider_file('file-vault','$KEYLOCATION');"
+    sudo -u "$PG_USER" psql -d "$DB_NAME" -p "$PG_PORT" -c "SELECT pg_tde_set_key_using_database_key_provider('test-db-master-key','file-vault');"
     sudo -u "$PG_USER" psql -p $PG_PORT -c "ALTER DATABASE $DB_NAME SET default_table_access_method='tde_heap';"
     sudo -u "$PG_USER" psql -p $PG_PORT -c "SELECT pg_reload_conf();"
 }

ci_scripts/configure-tde-server.sh → ci_scripts/configure-global-tde.sh

@@ -1,14 +1,14 @@
 #!/bin/bash
 set -e
 
-# This script is used to configure a TDE server for testing purposes.
-export TDE_MODE=1
-
 SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)"
 INSTALL_DIR="$SCRIPT_DIR/../../pginst"
+source $SCRIPT_DIR/env.sh
 
 cd "$SCRIPT_DIR/.."
 
+source "$SCRIPT_DIR/env.sh"
+export TDE_MODE=1
 export PATH=$INSTALL_DIR/bin:$PATH
 export DATA_DIR=$INSTALL_DIR/data
 export PGDATA="${1:-$DATA_DIR}"

ci_scripts/dump-typedefs.sh

@@ -1,33 +1,14 @@
 #!/bin/bash
 
 SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)"
-cd "$SCRIPT_DIR/../"
-
-if ! test -f src/backend/postgres; then
-  echo "src/backend/postgres doesn't exists, run make-build.sh first in debug mode"
-  exit 1
-fi
+cd "$SCRIPT_DIR/.."
 
 if ! test -f contrib/pg_tde/pg_tde.so; then
   echo "contrib/pg_tde/pg_tde.so doesn't exists, run make-build.sh first in debug mode"
   exit 1
 fi
 
-objdump -W src/backend/postgres |\
-    egrep -A3 DW_TAG_typedef |\
-    perl -e ' while (<>) { chomp; @flds = split;next unless (1 < @flds);\
-        next if $flds[0]  ne "DW_AT_name" && $flds[1] ne "DW_AT_name";\
-        next if $flds[-1] =~ /^DW_FORM_str/;\
-        print $flds[-1],"\n"; }'  |\
-    sort | uniq > percona.typedefs
-
-objdump -W contrib/pg_tde/pg_tde.so |\
-    egrep -A3 DW_TAG_typedef |\
-    perl -e ' while (<>) { chomp; @flds = split;next unless (1 < @flds);\
-        next if $flds[0]  ne "DW_AT_name" && $flds[1] ne "DW_AT_name";\
-        next if $flds[-1] =~ /^DW_FORM_str/;\
-        print $flds[-1],"\n"; }'  |\
-    sort | uniq > tde.typedefs
+src/tools/find_typedef contrib/pg_tde > pg_tde.typedefs
 
 # Combine with original typedefs
-cat percona.typedefs tde.typedefs src/tools/pgindent/typedefs.list | sort | uniq > combined.typedefs
+cat pg_tde.typedefs src/tools/pgindent/typedefs.list | sort -u > combined.typedefs

ci_scripts/env.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+export PERCONA_SERVER_VERSION=17.4.1

ci_scripts/make-build.sh

@@ -1,9 +1,19 @@
 #!/bin/bash
 
-export TDE_MODE=1
+ENABLE_COVERAGE=
+
+for arg in "$@"
+do
+    case "$arg" in
+        --enable-coverage)
+            ENABLE_COVERAGE="--enable-coverage"
+            shift;;
+    esac
+done
 
 SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)"
 INSTALL_DIR="$SCRIPT_DIR/../../pginst"
+source "$SCRIPT_DIR/env.sh"
 
 cd "$SCRIPT_DIR/.."
 
@@ -12,5 +22,5 @@ if [ "$1" = "debugoptimized" ]; then
     export CXXFLAGS="-O2"
 fi
 
-./configure --enable-debug --enable-cassert --enable-tap-tests --prefix=$INSTALL_DIR
+./configure --prefix="$INSTALL_DIR" --enable-debug --enable-cassert --enable-tap-tests $ENABLE_COVERAGE
 make install-world -j

ci_scripts/make-test-tde.sh → ci_scripts/make-test-global-tde.sh

@@ -1,16 +1,18 @@
 #!/bin/bash
 
 set -e
+ADD_FLAGS=
 
-export TDE_MODE=1
+for arg in "$@"
+do
+    case "$arg" in
+        --continue)
+            ADD_FLAGS="-k"
+            shift;;
+    esac
+done
 
 SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)"
-source $SCRIPT_DIR/configure-tde-server.sh
-
-ADD_FLAGS=
-
-if [ "$1" = "--continue" ]; then
-    ADD_FLAGS="-k"
-fi
+source "$SCRIPT_DIR/configure-global-tde.sh"
 
 EXTRA_REGRESS_OPTS="--extra-setup=$SCRIPT_DIR/tde_setup.sql" make -s installcheck-world $ADD_FLAGS

ci_scripts/make-test.sh

@@ -1,10 +1,25 @@
 #!/bin/bash
 
-export TDE_MODE=1
+set -e
+TDE_ONLY=0
 
-SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P)"
-INSTALL_DIR="$SCRIPT_DIR/../../pginst"
+for arg in "$@"
+do
+    case "$arg" in
+        --tde-only)
+            TDE_ONLY=1
+            shift;;
+    esac
+done
 
-cd "$SCRIPT_DIR/.."
+SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)"
+source "$SCRIPT_DIR/env.sh"
 
-make -s check-world
+if [ "$TDE_ONLY" -eq 1 ];
+then
+    cd "$SCRIPT_DIR/../contrib/pg_tde"
+    make -s check
+else
+    cd "$SCRIPT_DIR/.."
+    make -s check-world
+fi