From 167cc84917803297fb4458dab0375cf1cdcf3e7b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 2 May 2025 16:27:24 -0400 Subject: [PATCH 01/22] feat: if upgrade 17 -> 17 or 17-orioledb -> 17-orioledb do not run these queries --- .../admin_api_scripts/pg_upgrade_scripts/initiate.sh | 8 ++++++-- ansible/vars.yml | 6 +++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 92c1516a5..2b54557e8 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -43,8 +43,12 @@ POST_UPGRADE_EXTENSION_SCRIPT="/tmp/pg_upgrade/pg_upgrade_extensions.sql" POST_UPGRADE_POSTGRES_PERMS_SCRIPT="/tmp/pg_upgrade/pg_upgrade_postgres_perms.sql" OLD_PGVERSION=$(run_sql -A -t -c "SHOW server_version;") -SERVER_LC_COLLATE=$(run_sql -A -t -c "SHOW lc_collate;") -SERVER_LC_CTYPE=$(run_sql -A -t -c "SHOW lc_ctype;") +# Skip locale settings if both versions are PostgreSQL 17+ or 17-orioledb +if ! [[ (("$OLD_PGVERSION" =~ ^17.* || "$OLD_PGVERSION" == "17-orioledb") && ("$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb")) ]]; then + SERVER_LC_COLLATE=$(run_sql -A -t -c "SHOW lc_collate;") + SERVER_LC_CTYPE=$(run_sql -A -t -c "SHOW lc_ctype;") +fi + SERVER_ENCODING=$(run_sql -A -t -c "SHOW server_encoding;") POSTGRES_CONFIG_PATH="/etc/postgresql/postgresql.conf" diff --git a/ansible/vars.yml b/ansible/vars.yml index 4821ee48b..c6d15625e 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.079-orioledb" - postgres17: "17.4.1.029" - postgres15: "15.8.1.086" + postgresorioledb-17: "17.0.1.079-orioledb-pgup-1" + postgres17: "17.4.1.029-pgup-1" + postgres15: "15.8.1.086-pgup-1" # Non Postgres Extensions pgbouncer_release: "1.19.0" From bd7197175653bff2a4dab1067207785eedc6b405 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 5 May 2025 14:44:35 -0400 Subject: [PATCH 02/22] feat: handle all cases of SERVER_LC_COLLATE and SERVER_LC_CTYPE --- .../pg_upgrade_scripts/initiate.sh | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 2b54557e8..58b78ea37 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -41,7 +41,7 @@ LOG_FILE="/var/log/pg-upgrade-initiate.log" POST_UPGRADE_EXTENSION_SCRIPT="/tmp/pg_upgrade/pg_upgrade_extensions.sql" POST_UPGRADE_POSTGRES_PERMS_SCRIPT="/tmp/pg_upgrade/pg_upgrade_postgres_perms.sql" -OLD_PGVERSION=$(run_sql -A -t -c "SHOW server_version;") +OLD_PGVERSION=$(pg_config --version | sed 's/PostgreSQL \([0-9]*\.[0-9]*\).*/\1/') # Skip locale settings if both versions are PostgreSQL 17+ or 17-orioledb if ! [[ (("$OLD_PGVERSION" =~ ^17.* || "$OLD_PGVERSION" == "17-orioledb") && ("$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb")) ]]; then @@ -398,7 +398,11 @@ function initiate_upgrade { rm -rf "${PGDATANEW:?}/" if [ "$IS_NIX_UPGRADE" = "true" ]; then - LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --encoding=$SERVER_ENCODING --lc-collate=$SERVER_LC_COLLATE --lc-ctype=$SERVER_LC_CTYPE -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres + if [[ "$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb" ]]; then + LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --encoding=$SERVER_ENCODING --locale-provider=icu --icu-locale=en_US.UTF-8 -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres + else + LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --encoding=$SERVER_ENCODING --lc-collate=$SERVER_LC_COLLATE --lc-ctype=$SERVER_LC_CTYPE -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres + fi else su -c "$PGBINNEW/initdb -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres fi @@ -431,7 +435,12 @@ EOF if [ "$IS_NIX_BASED_SYSTEM" = "true" ]; then UPGRADE_COMMAND=". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $UPGRADE_COMMAND" fi - GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres + + if [[ "$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb" ]]; then + GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres + else + GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres + fi echo "10. Stopping postgres; running pg_upgrade" # Extra work to ensure postgres is actually stopped @@ -447,9 +456,12 @@ EOF CI_stop_postgres fi - GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND" -s "$SHELL" postgres + if [[ "$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb" ]]; then + GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND" -s "$SHELL" postgres + else + GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND" -s "$SHELL" postgres + fi - # copying custom configurations echo "11. Copying custom configurations" mkdir -p "$MOUNT_POINT/conf" cp -R /etc/postgresql-custom/* "$MOUNT_POINT/conf/" From 35116f1f521a3fde885f2df80bb3698a2afe3376 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 6 May 2025 10:28:17 -0400 Subject: [PATCH 03/22] fix: explixit set on 17/oriole --- ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 58b78ea37..98a2b24f6 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -399,7 +399,7 @@ function initiate_upgrade { if [ "$IS_NIX_UPGRADE" = "true" ]; then if [[ "$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb" ]]; then - LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --encoding=$SERVER_ENCODING --locale-provider=icu --icu-locale=en_US.UTF-8 -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres + LC_ALL=en_US.UTF-8 LC_CTYPE=en_US.UTF-8 LC_COLLATE=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --encoding=$SERVER_ENCODING --locale-provider=icu --icu-locale=en_US.UTF-8 -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres else LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --encoding=$SERVER_ENCODING --lc-collate=$SERVER_LC_COLLATE --lc-ctype=$SERVER_LC_CTYPE -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres fi From 096a6b5cb537bd9a5e5188e68da61dbfa046dd91 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 6 May 2025 14:49:10 -0400 Subject: [PATCH 04/22] feat: handling max_slot_wal_keep_size for pg 17 was needed as well --- .../admin_api_scripts/pg_upgrade_scripts/complete.sh | 12 ++++++++++++ .../admin_api_scripts/pg_upgrade_scripts/initiate.sh | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh index 3aeff263c..61a8229da 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh @@ -22,6 +22,18 @@ function cleanup { ship_logs "$LOG_FILE" || true + # Restore max_slot_wal_keep_size to 4096 only if both versions were PostgreSQL 17+ + if [[ ("$OLD_PGVERSION" =~ ^17.* || "$OLD_PGVERSION" == "17-orioledb") && ("$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb") ]]; then + sed -i 's/max_slot_wal_keep_size = -1/max_slot_wal_keep_size = 4096/' /etc/postgresql/postgresql.conf + fi + + # Restart postgres to apply any configuration changes + if [ -z "$IS_CI" ]; then + systemctl restart postgresql + else + CI_start_postgres + fi + exit "$EXIT_CODE" } diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 98a2b24f6..8e63eb8a0 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -397,6 +397,11 @@ function initiate_upgrade { chown -R postgres:postgres "$MOUNT_POINT/" rm -rf "${PGDATANEW:?}/" + # Change max_slot_wal_keep_size to -1 for upgrade only if both versions are PostgreSQL 17+ + if [[ ("$OLD_PGVERSION" =~ ^17.* || "$OLD_PGVERSION" == "17-orioledb") && ("$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb") ]]; then + sed -i 's/max_slot_wal_keep_size = [0-9]*/max_slot_wal_keep_size = -1/' /etc/postgresql/postgresql.conf + fi + if [ "$IS_NIX_UPGRADE" = "true" ]; then if [[ "$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb" ]]; then LC_ALL=en_US.UTF-8 LC_CTYPE=en_US.UTF-8 LC_COLLATE=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --encoding=$SERVER_ENCODING --locale-provider=icu --icu-locale=en_US.UTF-8 -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres From 3b57d25cc05f6b19830d5c5b71773ddab1b72f65 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 6 May 2025 14:57:20 -0400 Subject: [PATCH 05/22] feat: binary upgrades require max_slot_wal_keep_size to be -1 during binary upgrade --- .../files/admin_api_scripts/pg_upgrade_scripts/complete.sh | 6 ++---- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh index 61a8229da..bc0fa5f4e 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh @@ -22,10 +22,8 @@ function cleanup { ship_logs "$LOG_FILE" || true - # Restore max_slot_wal_keep_size to 4096 only if both versions were PostgreSQL 17+ - if [[ ("$OLD_PGVERSION" =~ ^17.* || "$OLD_PGVERSION" == "17-orioledb") && ("$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb") ]]; then - sed -i 's/max_slot_wal_keep_size = -1/max_slot_wal_keep_size = 4096/' /etc/postgresql/postgresql.conf - fi + # Restore max_slot_wal_keep_size to 4096 after binary upgrade + sed -i 's/max_slot_wal_keep_size = -1/max_slot_wal_keep_size = 4096/' /etc/postgresql/postgresql.conf # Restart postgres to apply any configuration changes if [ -z "$IS_CI" ]; then diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 8e63eb8a0..81abe5fb0 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -397,10 +397,8 @@ function initiate_upgrade { chown -R postgres:postgres "$MOUNT_POINT/" rm -rf "${PGDATANEW:?}/" - # Change max_slot_wal_keep_size to -1 for upgrade only if both versions are PostgreSQL 17+ - if [[ ("$OLD_PGVERSION" =~ ^17.* || "$OLD_PGVERSION" == "17-orioledb") && ("$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb") ]]; then - sed -i 's/max_slot_wal_keep_size = [0-9]*/max_slot_wal_keep_size = -1/' /etc/postgresql/postgresql.conf - fi + # Change max_slot_wal_keep_size to -1 for binary upgrade + sed -i 's/max_slot_wal_keep_size = [0-9]*/max_slot_wal_keep_size = -1/' /etc/postgresql/postgresql.conf if [ "$IS_NIX_UPGRADE" = "true" ]; then if [[ "$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb" ]]; then From f442e52b2011fec6ad17f963c1c976a687a27b13 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 6 May 2025 16:07:56 -0400 Subject: [PATCH 06/22] fix: Better to override that during the upgrade process by specifying the option directly when pg_upgrade is executed (-c) --- .../files/admin_api_scripts/pg_upgrade_scripts/complete.sh | 3 --- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 7 +++---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh index bc0fa5f4e..482521648 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh @@ -22,9 +22,6 @@ function cleanup { ship_logs "$LOG_FILE" || true - # Restore max_slot_wal_keep_size to 4096 after binary upgrade - sed -i 's/max_slot_wal_keep_size = -1/max_slot_wal_keep_size = 4096/' /etc/postgresql/postgresql.conf - # Restart postgres to apply any configuration changes if [ -z "$IS_CI" ]; then systemctl restart postgresql diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 81abe5fb0..8b6910b30 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -397,9 +397,6 @@ function initiate_upgrade { chown -R postgres:postgres "$MOUNT_POINT/" rm -rf "${PGDATANEW:?}/" - # Change max_slot_wal_keep_size to -1 for binary upgrade - sed -i 's/max_slot_wal_keep_size = [0-9]*/max_slot_wal_keep_size = -1/' /etc/postgresql/postgresql.conf - if [ "$IS_NIX_UPGRADE" = "true" ]; then if [[ "$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb" ]]; then LC_ALL=en_US.UTF-8 LC_CTYPE=en_US.UTF-8 LC_COLLATE=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --encoding=$SERVER_ENCODING --locale-provider=icu --icu-locale=en_US.UTF-8 -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres @@ -430,8 +427,10 @@ $(cat /etc/postgresql/pg_hba.conf)" > /etc/postgresql/pg_hba.conf --jobs="${WORKERS}" -r \ --old-options='-c config_file=${POSTGRES_CONFIG_PATH}' \ --old-options="-c shared_preload_libraries='${SHARED_PRELOAD_LIBRARIES}'" \ + --old-options="-c max_slot_wal_keep_size=-1" \ --new-options="-c data_directory=${PGDATANEW}" \ - --new-options="-c shared_preload_libraries='${SHARED_PRELOAD_LIBRARIES}'" + --new-options="-c shared_preload_libraries='${SHARED_PRELOAD_LIBRARIES}'" \ + --new-options="-c max_slot_wal_keep_size=-1" EOF ) From d54f361b6ac2c97bef1a5ccf8af182ae45f40d44 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 7 May 2025 07:59:14 -0400 Subject: [PATCH 07/22] fix: cover only pg 17 --- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 8b6910b30..522f8d71c 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -438,7 +438,7 @@ EOF UPGRADE_COMMAND=". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $UPGRADE_COMMAND" fi - if [[ "$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb" ]]; then + if [[ "$PGVERSION" =~ ^17.* ]]; then GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres else GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres @@ -458,7 +458,7 @@ EOF CI_stop_postgres fi - if [[ "$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb" ]]; then + if [[ "$PGVERSION" =~ ^17.* ]]; then GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND" -s "$SHELL" postgres else GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND" -s "$SHELL" postgres From e2028b2d726a2d28953c949e39299a9a042143d0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 7 May 2025 10:01:05 -0400 Subject: [PATCH 08/22] fix: rm oriole handling --- ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 522f8d71c..566470fc2 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -398,7 +398,7 @@ function initiate_upgrade { rm -rf "${PGDATANEW:?}/" if [ "$IS_NIX_UPGRADE" = "true" ]; then - if [[ "$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb" ]]; then + if [[ "$PGVERSION" =~ ^17.* ]]; then LC_ALL=en_US.UTF-8 LC_CTYPE=en_US.UTF-8 LC_COLLATE=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --encoding=$SERVER_ENCODING --locale-provider=icu --icu-locale=en_US.UTF-8 -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres else LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $PGBINNEW/initdb --encoding=$SERVER_ENCODING --lc-collate=$SERVER_LC_COLLATE --lc-ctype=$SERVER_LC_CTYPE -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres From 68e4a8511198ff70b4ab3389f074c841ebae381a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 7 May 2025 11:18:37 -0400 Subject: [PATCH 09/22] fix: do not need max_slot_wal_keep_size on old version --- ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 566470fc2..ba0c3c162 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -427,7 +427,6 @@ $(cat /etc/postgresql/pg_hba.conf)" > /etc/postgresql/pg_hba.conf --jobs="${WORKERS}" -r \ --old-options='-c config_file=${POSTGRES_CONFIG_PATH}' \ --old-options="-c shared_preload_libraries='${SHARED_PRELOAD_LIBRARIES}'" \ - --old-options="-c max_slot_wal_keep_size=-1" \ --new-options="-c data_directory=${PGDATANEW}" \ --new-options="-c shared_preload_libraries='${SHARED_PRELOAD_LIBRARIES}'" \ --new-options="-c max_slot_wal_keep_size=-1" From 556baec936967b89816d5d2b099a326ab5857f8a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 7 May 2025 13:19:07 -0400 Subject: [PATCH 10/22] fix: temp config on new-options too --- ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index ba0c3c162..da303ad1e 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -429,6 +429,7 @@ $(cat /etc/postgresql/pg_hba.conf)" > /etc/postgresql/pg_hba.conf --old-options="-c shared_preload_libraries='${SHARED_PRELOAD_LIBRARIES}'" \ --new-options="-c data_directory=${PGDATANEW}" \ --new-options="-c shared_preload_libraries='${SHARED_PRELOAD_LIBRARIES}'" \ + --new-options='-c config_file=${TEMP_POSTGRES_CONFIG_PATH}' \ --new-options="-c max_slot_wal_keep_size=-1" EOF ) From 2f9044bae8e56f9740fe27312a3b40c46dd35be4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 7 May 2025 15:17:25 -0400 Subject: [PATCH 11/22] fix: remove unbound var --- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 1 - ansible/vars.yml | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index da303ad1e..ba0c3c162 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -429,7 +429,6 @@ $(cat /etc/postgresql/pg_hba.conf)" > /etc/postgresql/pg_hba.conf --old-options="-c shared_preload_libraries='${SHARED_PRELOAD_LIBRARIES}'" \ --new-options="-c data_directory=${PGDATANEW}" \ --new-options="-c shared_preload_libraries='${SHARED_PRELOAD_LIBRARIES}'" \ - --new-options='-c config_file=${TEMP_POSTGRES_CONFIG_PATH}' \ --new-options="-c max_slot_wal_keep_size=-1" EOF ) diff --git a/ansible/vars.yml b/ansible/vars.yml index c6d15625e..a2f235b6e 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,15 @@ postgres_major: # Full version strings for each major version postgres_release: +<<<<<<< HEAD postgresorioledb-17: "17.0.1.079-orioledb-pgup-1" postgres17: "17.4.1.029-pgup-1" postgres15: "15.8.1.086-pgup-1" +======= + postgresorioledb-17: "17.0.1.078-orioledb-upgrade-8" + postgres17: "17.4.1.028-upgrade-8" + postgres15: "15.8.1.085-upgrade-8" +>>>>>>> 1ca84730 (fix: remove unbound var) # Non Postgres Extensions pgbouncer_release: "1.19.0" From e734bfbbb8809dcaba6c6c72d4037a528af7687f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 7 May 2025 17:10:12 -0400 Subject: [PATCH 12/22] chore: remove complete.sh change that should not have been committed --- .../files/admin_api_scripts/pg_upgrade_scripts/complete.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh index 482521648..3aeff263c 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh @@ -22,13 +22,6 @@ function cleanup { ship_logs "$LOG_FILE" || true - # Restart postgres to apply any configuration changes - if [ -z "$IS_CI" ]; then - systemctl restart postgresql - else - CI_start_postgres - fi - exit "$EXIT_CODE" } From e07ec085bca101b60e60b76976cb914f0a2273f0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 7 May 2025 17:14:25 -0400 Subject: [PATCH 13/22] chore: bump for testing --- ansible/vars.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index a2f235b6e..c6d15625e 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,15 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: -<<<<<<< HEAD postgresorioledb-17: "17.0.1.079-orioledb-pgup-1" postgres17: "17.4.1.029-pgup-1" postgres15: "15.8.1.086-pgup-1" -======= - postgresorioledb-17: "17.0.1.078-orioledb-upgrade-8" - postgres17: "17.4.1.028-upgrade-8" - postgres15: "15.8.1.085-upgrade-8" ->>>>>>> 1ca84730 (fix: remove unbound var) # Non Postgres Extensions pgbouncer_release: "1.19.0" From f067318b5cf688acdbd0c7b2005adf1510a82c56 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 9 May 2025 12:57:42 -0400 Subject: [PATCH 14/22] chore: stash code --- .../pg_upgrade_scripts/initiate.sh | 68 +++++++++-- nix/docs/testing-pg-upgrade-scripts.md | 115 ++++++++++++++++++ 2 files changed, 171 insertions(+), 12 deletions(-) create mode 100644 nix/docs/testing-pg-upgrade-scripts.md diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index ba0c3c162..82353e7ea 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -52,7 +52,7 @@ fi SERVER_ENCODING=$(run_sql -A -t -c "SHOW server_encoding;") POSTGRES_CONFIG_PATH="/etc/postgresql/postgresql.conf" -PGBINOLD="/usr/lib/postgresql/bin" +PGBINOLD="$(pg_config --bindir)" PG_UPGRADE_BIN_DIR="/tmp/pg_upgrade_bin/$PGVERSION" NIX_INSTALLER_PATH="/tmp/persistent/nix-installer" @@ -242,6 +242,10 @@ function initiate_upgrade { fi PGDATAOLD=$(cat "$POSTGRES_CONFIG_PATH" | grep data_directory | sed "s/data_directory = '\(.*\)'.*/\1/") + # Resolve symlink if it exists + if [ -L "$PGDATAOLD" ]; then + PGDATAOLD=$(readlink -f "$PGDATAOLD") + fi PGDATANEW="$MOUNT_POINT/pgdata" @@ -255,7 +259,12 @@ function initiate_upgrade { if [ -n "$IS_LOCAL_UPGRADE" ]; then mkdir -p "$PG_UPGRADE_BIN_DIR" mkdir -p /tmp/persistent/ - echo "a7189a68ed4ea78c1e73991b5f271043636cf074" > "$PG_UPGRADE_BIN_DIR/nix_flake_version" + # Use NIX_FLAKE_VERSION if set, otherwise use default + if [ -n "$NIX_FLAKE_VERSION" ]; then + echo "$NIX_FLAKE_VERSION" > "$PG_UPGRADE_BIN_DIR/nix_flake_version" + else + echo "a7189a68ed4ea78c1e73991b5f271043636cf074" > "$PG_UPGRADE_BIN_DIR/nix_flake_version" + fi tar -czf "/tmp/persistent/pg_upgrade_bin.tar.gz" -C "/tmp/pg_upgrade_bin" . rm -rf /tmp/pg_upgrade_bin/ fi @@ -320,6 +329,7 @@ function initiate_upgrade { # upgrade job outputs a log in the cwd; needs write permissions mkdir -p /tmp/pg_upgrade/ chown -R postgres:postgres /tmp/pg_upgrade/ + chmod 0700 /tmp/pg_upgrade/ cd /tmp/pg_upgrade/ # Fixing erros generated by previous dpkg executions (package upgrades et co) @@ -417,25 +427,51 @@ $(cat /etc/postgresql/pg_hba.conf)" > /etc/postgresql/pg_hba.conf run_sql -c "select pg_reload_conf();" fi + # Build the base upgrade command UPGRADE_COMMAND=$(cat <> "$TMP_CONFIG" + chown postgres:postgres "$TMP_CONFIG" + + UPGRADE_COMMAND="$UPGRADE_COMMAND \ + --old-options='-c config_file=$TMP_CONFIG' \ + --new-options='-c max_slot_wal_keep_size=-1' \ + --new-options='-c unix_socket_directories=/tmp/pg_upgrade'" + else + UPGRADE_COMMAND="$UPGRADE_COMMAND \ + --old-options='-c config_file=${POSTGRES_CONFIG_PATH}'" + fi + if [ "$IS_NIX_BASED_SYSTEM" = "true" ]; then - UPGRADE_COMMAND=". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $UPGRADE_COMMAND" - fi + # Ensure nix environment is loaded for both old and new instances + UPGRADE_COMMAND=". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && PGBINOLD=${PGBINOLD} $UPGRADE_COMMAND" + fi + + # Set max_slot_wal_keep_size in environment for PG17 upgrades + if [[ "$PGVERSION" =~ ^17.* ]]; then + export PGDATA_UPGRADE_MAX_SLOT_WAL_KEEP_SIZE=-1 + fi if [[ "$PGVERSION" =~ ^17.* ]]; then GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres @@ -457,6 +493,14 @@ EOF CI_stop_postgres fi + # Create and set permissions on socket directory for PG17 upgrades + if [[ "$PGVERSION" =~ ^17.* ]]; then + mkdir -p /tmp/pg_upgrade + chown postgres:postgres /tmp/pg_upgrade + chmod 0700 /tmp/pg_upgrade + fi + + # Start the old PostgreSQL instance with version-specific options if [[ "$PGVERSION" =~ ^17.* ]]; then GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND" -s "$SHELL" postgres else diff --git a/nix/docs/testing-pg-upgrade-scripts.md b/nix/docs/testing-pg-upgrade-scripts.md new file mode 100644 index 000000000..49c966b5f --- /dev/null +++ b/nix/docs/testing-pg-upgrade-scripts.md @@ -0,0 +1,115 @@ +# Testing PostgreSQL Upgrade Scripts + +This document describes how to test changes to the PostgreSQL upgrade scripts on a running machine. + +## Prerequisites + +- A running PostgreSQL instance +- Access to the Supabase Postgres repository +- Permissions to run GitHub Actions workflows +- ssh access to the ec2 instance + +## Development Workflow + +1. **Make Changes to Upgrade Scripts** + - Make your changes to the scripts in `ansible/files/admin_api_scripts/pg_upgrade_scripts/` + - Commit and push your changes to your feature branch + - For quick testing, you can also edit the script directly on the server at `/etc/adminapi/pg_upgrade_scripts/initiate.sh` + +2. **Publish Script Changes** (Only needed for deploying to new instances) + - Go to [publish-nix-pgupgrade-scripts.yml](https://github.com/supabase/postgres/actions/workflows/publish-nix-pgupgrade-scripts.yml) + - Click "Run workflow" + - Select your branch + - Run the workflow + +3. **Publish Binary Flake Version** (Only needed for deploying to new instances) + - Go to [publish-nix-pgupgrade-bin-flake-version.yml](https://github.com/supabase/postgres/actions/workflows/publish-nix-pgupgrade-bin-flake-version.yml) + - Click "Run workflow" + - Select your branch + - Run the workflow + - Note: Make sure the flake version includes the PostgreSQL version you're testing (e.g., 17) + +4. **Test on Running Machine** + ssh into the machine + ```bash + # Stop PostgreSQL + sudo systemctl stop postgresql + + # Run the upgrade script in local mode with your desired flake version + sudo NIX_FLAKE_VERSION="your-flake-version-here" IS_LOCAL_UPGRADE=true /etc/adminapi/pg_upgrade_scripts/initiate.sh 17 + ``` + Note: This will use the version of the script that exists at `/etc/adminapi/pg_upgrade_scripts/initiate.sh` on the server. + The script should be run as the ubuntu user with sudo privileges. The script will handle switching to the postgres user when needed. + + In local mode: + - The script at `/etc/adminapi/pg_upgrade_scripts/initiate.sh` will be used (your edited version) + - Only the PostgreSQL binaries will be downloaded from the specified flake version + - No new upgrade scripts will be downloaded + - You can override the flake version by setting the NIX_FLAKE_VERSION environment variable + - If NIX_FLAKE_VERSION is not set, it will use the default flake version + +5. **Monitor Progress** + ```bash + # Watch the upgrade log + tail -f /var/log/pg-upgrade-initiate.log + ``` + +6. **Check Results** + In local mode, the script will: + - Create a new data directory at `/data_migration/pgdata` + - Run pg_upgrade to test the upgrade process + - Generate SQL files in `/data_migration/sql/` for any needed post-upgrade steps + - Log the results in `/var/log/pg-upgrade-initiate.log` + + To verify success: + ```bash + # Check the upgrade log for completion + grep "Upgrade complete" /var/log/pg-upgrade-initiate.log + + # Check for any generated SQL files + ls -l /data_migration/sql/ + + # Check the new data directory + ls -l /data_migration/pgdata/ + ``` + + Note: The instance will not be upgraded to the new version in local mode. This is just a test run to verify the upgrade process works correctly. + +## Important Notes + +- The `IS_LOCAL_UPGRADE=true` flag makes the script run in the foreground and skip disk mounting steps +- The script will use the existing data directory +- All output is logged to `/var/log/pg-upgrade-initiate.log` +- The script will automatically restart PostgreSQL after completion or failure +- For testing, you can edit the script directly on the server - the GitHub Actions workflows are only needed for deploying to new instances +- Run the script as the ubuntu user with sudo privileges - the script will handle user switching internally +- Local mode is for testing only - it will not actually upgrade the instance +- The Nix flake version must include the PostgreSQL version you're testing (e.g., 17) +- In local mode, only the PostgreSQL binaries are downloaded from the flake - the upgrade scripts are used from the local filesystem +- You can override the flake version by setting the NIX_FLAKE_VERSION environment variable when running the script + +## Troubleshooting + +If the upgrade fails: +1. Check the logs at `/var/log/pg-upgrade-initiate.log` +2. Look for any error messages in the PostgreSQL logs +3. The script will attempt to clean up and restore the original state +4. If you see an error about missing Nix flake attributes, make sure the flake version includes the PostgreSQL version you're testing + +Common Errors: +- `error: flake 'github:supabase/postgres/...' does not provide attribute 'packages.aarch64-linux.psql_17/bin'` + - This means the Nix flake version doesn't include PostgreSQL 17 binaries + - You need to specify a flake version that includes your target version + - You can find valid flake versions by looking at the commit history of the publish-nix-pgupgrade-bin-flake-version.yml workflow + +## Cleanup + +After testing: +1. The script will automatically clean up temporary files +2. PostgreSQL will be restarted +3. The original configuration will be restored + +## References + +- [publish-nix-pgupgrade-scripts.yml](https://github.com/supabase/postgres/actions/workflows/publish-nix-pgupgrade-scripts.yml) +- [publish-nix-pgupgrade-bin-flake-version.yml](https://github.com/supabase/postgres/actions/workflows/publish-nix-pgupgrade-bin-flake-version.yml) \ No newline at end of file From 8ea53516fa46acf2958112cf50fb0f70b493730d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 9 May 2025 14:14:05 -0400 Subject: [PATCH 15/22] feat: working pg 17 upgrade --- .../pg_upgrade_scripts/initiate.sh | 80 +++++++------------ 1 file changed, 29 insertions(+), 51 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 82353e7ea..b3b232f85 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -52,7 +52,7 @@ fi SERVER_ENCODING=$(run_sql -A -t -c "SHOW server_encoding;") POSTGRES_CONFIG_PATH="/etc/postgresql/postgresql.conf" -PGBINOLD="$(pg_config --bindir)" +PGBINOLD="/usr/lib/postgresql/bin" PG_UPGRADE_BIN_DIR="/tmp/pg_upgrade_bin/$PGVERSION" NIX_INSTALLER_PATH="/tmp/persistent/nix-installer" @@ -242,10 +242,6 @@ function initiate_upgrade { fi PGDATAOLD=$(cat "$POSTGRES_CONFIG_PATH" | grep data_directory | sed "s/data_directory = '\(.*\)'.*/\1/") - # Resolve symlink if it exists - if [ -L "$PGDATAOLD" ]; then - PGDATAOLD=$(readlink -f "$PGDATAOLD") - fi PGDATANEW="$MOUNT_POINT/pgdata" @@ -259,12 +255,12 @@ function initiate_upgrade { if [ -n "$IS_LOCAL_UPGRADE" ]; then mkdir -p "$PG_UPGRADE_BIN_DIR" mkdir -p /tmp/persistent/ - # Use NIX_FLAKE_VERSION if set, otherwise use default - if [ -n "$NIX_FLAKE_VERSION" ]; then + if [ -n "$NIX_FLAKE_VERSION" ]; then echo "$NIX_FLAKE_VERSION" > "$PG_UPGRADE_BIN_DIR/nix_flake_version" else echo "a7189a68ed4ea78c1e73991b5f271043636cf074" > "$PG_UPGRADE_BIN_DIR/nix_flake_version" fi + tar -czf "/tmp/persistent/pg_upgrade_bin.tar.gz" -C "/tmp/pg_upgrade_bin" . rm -rf /tmp/pg_upgrade_bin/ fi @@ -329,7 +325,6 @@ function initiate_upgrade { # upgrade job outputs a log in the cwd; needs write permissions mkdir -p /tmp/pg_upgrade/ chown -R postgres:postgres /tmp/pg_upgrade/ - chmod 0700 /tmp/pg_upgrade/ cd /tmp/pg_upgrade/ # Fixing erros generated by previous dpkg executions (package upgrades et co) @@ -415,6 +410,7 @@ function initiate_upgrade { fi else su -c "$PGBINNEW/initdb -L $PGSHARENEW -D $PGDATANEW/ --username=supabase_admin" -s "$SHELL" postgres + fi # This line avoids the need to supply the supabase_admin password on the old @@ -427,51 +423,39 @@ $(cat /etc/postgresql/pg_hba.conf)" > /etc/postgresql/pg_hba.conf run_sql -c "select pg_reload_conf();" fi - # Build the base upgrade command + TMP_CONFIG="/tmp/pg_upgrade/postgresql.conf" + cp "$POSTGRES_CONFIG_PATH" "$TMP_CONFIG" + + # Check if max_slot_wal_keep_size exists in the config + if grep -q "max_slot_wal_keep_size" "$TMP_CONFIG"; then + # Find and replace the existing setting + sed -i 's/^\s*max_slot_wal_keep_size\s*=.*$/max_slot_wal_keep_size = -1/' "$TMP_CONFIG" + else + # Add the setting if not found + echo "max_slot_wal_keep_size = -1" >> "$TMP_CONFIG" + fi + + chown postgres:postgres "$TMP_CONFIG" + UPGRADE_COMMAND=$(cat <> "$TMP_CONFIG" - chown postgres:postgres "$TMP_CONFIG" - - UPGRADE_COMMAND="$UPGRADE_COMMAND \ - --old-options='-c config_file=$TMP_CONFIG' \ - --new-options='-c max_slot_wal_keep_size=-1' \ - --new-options='-c unix_socket_directories=/tmp/pg_upgrade'" - else - UPGRADE_COMMAND="$UPGRADE_COMMAND \ - --old-options='-c config_file=${POSTGRES_CONFIG_PATH}'" - fi - if [ "$IS_NIX_BASED_SYSTEM" = "true" ]; then - # Ensure nix environment is loaded for both old and new instances - UPGRADE_COMMAND=". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && PGBINOLD=${PGBINOLD} $UPGRADE_COMMAND" - fi - - # Set max_slot_wal_keep_size in environment for PG17 upgrades - if [[ "$PGVERSION" =~ ^17.* ]]; then - export PGDATA_UPGRADE_MAX_SLOT_WAL_KEEP_SIZE=-1 - fi + UPGRADE_COMMAND=". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && $UPGRADE_COMMAND" + fi if [[ "$PGVERSION" =~ ^17.* ]]; then GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND --check" -s "$SHELL" postgres @@ -493,13 +477,6 @@ EOF CI_stop_postgres fi - # Create and set permissions on socket directory for PG17 upgrades - if [[ "$PGVERSION" =~ ^17.* ]]; then - mkdir -p /tmp/pg_upgrade - chown postgres:postgres /tmp/pg_upgrade - chmod 0700 /tmp/pg_upgrade - fi - # Start the old PostgreSQL instance with version-specific options if [[ "$PGVERSION" =~ ^17.* ]]; then GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND" -s "$SHELL" postgres @@ -507,6 +484,7 @@ EOF GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins LC_ALL=en_US.UTF-8 LC_CTYPE=$SERVER_LC_CTYPE LC_COLLATE=$SERVER_LC_COLLATE LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive su -pc "$UPGRADE_COMMAND" -s "$SHELL" postgres fi + # copying custom configurations echo "11. Copying custom configurations" mkdir -p "$MOUNT_POINT/conf" cp -R /etc/postgresql-custom/* "$MOUNT_POINT/conf/" From dce6cfb89801e835dd48ce20a7e45ef3f8792899 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 9 May 2025 19:37:06 -0400 Subject: [PATCH 16/22] feat: pg 15 handling --- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index b3b232f85..3a1f770d2 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -435,6 +435,11 @@ $(cat /etc/postgresql/pg_hba.conf)" > /etc/postgresql/pg_hba.conf echo "max_slot_wal_keep_size = -1" >> "$TMP_CONFIG" fi + # Remove db_user_namespace if upgrading from PG15 + if [[ "$OLD_PGVERSION" =~ ^15.* ]]; then + sed -i '/^db_user_namespace/d' "$TMP_CONFIG" + fi + chown postgres:postgres "$TMP_CONFIG" UPGRADE_COMMAND=$(cat < Date: Sat, 10 May 2025 13:00:33 -0400 Subject: [PATCH 17/22] feat: rm oriole handling, refine 15 -> 17 config --- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 3a1f770d2..1c5612e34 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -43,8 +43,8 @@ POST_UPGRADE_EXTENSION_SCRIPT="/tmp/pg_upgrade/pg_upgrade_extensions.sql" POST_UPGRADE_POSTGRES_PERMS_SCRIPT="/tmp/pg_upgrade/pg_upgrade_postgres_perms.sql" OLD_PGVERSION=$(pg_config --version | sed 's/PostgreSQL \([0-9]*\.[0-9]*\).*/\1/') -# Skip locale settings if both versions are PostgreSQL 17+ or 17-orioledb -if ! [[ (("$OLD_PGVERSION" =~ ^17.* || "$OLD_PGVERSION" == "17-orioledb") && ("$PGVERSION" =~ ^17.* || "$PGVERSION" == "17-orioledb")) ]]; then +# Skip locale settings if both versions are PostgreSQL 17+ +if ! [[ "$OLD_PGVERSION" =~ ^17.* && "$PGVERSION" =~ ^17.* ]]; then SERVER_LC_COLLATE=$(run_sql -A -t -c "SHOW lc_collate;") SERVER_LC_CTYPE=$(run_sql -A -t -c "SHOW lc_ctype;") fi @@ -436,7 +436,7 @@ $(cat /etc/postgresql/pg_hba.conf)" > /etc/postgresql/pg_hba.conf fi # Remove db_user_namespace if upgrading from PG15 - if [[ "$OLD_PGVERSION" =~ ^15.* ]]; then + if [[ "$OLD_PGVERSION" =~ ^15.* && "$PGVERSION" =~ ^17.* ]]; then sed -i '/^db_user_namespace/d' "$TMP_CONFIG" fi From 921e3cad3c1308568a8bad6a8884b1a1eca8f6dc Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Sat, 10 May 2025 21:14:31 -0400 Subject: [PATCH 18/22] feat: make sure old pg stops if not force stop --- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 1c5612e34..08ac37233 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -478,6 +478,13 @@ EOF sleep 3 systemctl stop postgresql + + # Additional check to ensure postgres is really stopped + if [ -f "${PGDATAOLD}/postmaster.pid" ]; then + echo "PostgreSQL still running, forcing stop..." + kill -9 $(head -n 1 "${PGDATAOLD}/postmaster.pid") || true + rm -f "${PGDATAOLD}/postmaster.pid" + fi else CI_stop_postgres fi From 0fc162384cff184beed776adc07f56c08f5d3d64 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Sat, 10 May 2025 21:15:32 -0400 Subject: [PATCH 19/22] chore: bump version --- ansible/vars.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ansible/vars.yml b/ansible/vars.yml index c6d15625e..60c62d0b6 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,15 @@ postgres_major: # Full version strings for each major version postgres_release: +<<<<<<< HEAD postgresorioledb-17: "17.0.1.079-orioledb-pgup-1" postgres17: "17.4.1.029-pgup-1" postgres15: "15.8.1.086-pgup-1" +======= + postgresorioledb-17: "17.0.1.079-orioledb-upgrade-13" + postgres17: "17.4.1.029-upgrade-13" + postgres15: "15.8.1.086-upgrade-13" +>>>>>>> c27f9870 (chore: bump version) # Non Postgres Extensions pgbouncer_release: "1.19.0" From 8ca4ab0d39743b810485ed52847f624c79c8140f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 12 May 2025 09:29:00 -0400 Subject: [PATCH 20/22] chore: cleanup + bump version for test --- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 3 ++- ansible/vars.yml | 6 ------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 08ac37233..b576b6fd7 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -482,7 +482,8 @@ EOF # Additional check to ensure postgres is really stopped if [ -f "${PGDATAOLD}/postmaster.pid" ]; then echo "PostgreSQL still running, forcing stop..." - kill -9 $(head -n 1 "${PGDATAOLD}/postmaster.pid") || true + pid=$(head -n 1 "${PGDATAOLD}/postmaster.pid") + kill -9 "$pid" || true rm -f "${PGDATAOLD}/postmaster.pid" fi else diff --git a/ansible/vars.yml b/ansible/vars.yml index 60c62d0b6..c6d15625e 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,15 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: -<<<<<<< HEAD postgresorioledb-17: "17.0.1.079-orioledb-pgup-1" postgres17: "17.4.1.029-pgup-1" postgres15: "15.8.1.086-pgup-1" -======= - postgresorioledb-17: "17.0.1.079-orioledb-upgrade-13" - postgres17: "17.4.1.029-upgrade-13" - postgres15: "15.8.1.086-upgrade-13" ->>>>>>> c27f9870 (chore: bump version) # Non Postgres Extensions pgbouncer_release: "1.19.0" From 28cb728c5a60b374940ea0c600a3d11cbacc574c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 15 May 2025 09:35:58 -0400 Subject: [PATCH 21/22] fix: rollback to working version with fix from divit --- .../pg_upgrade_scripts/initiate.sh | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index b576b6fd7..85b025be0 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -427,13 +427,8 @@ $(cat /etc/postgresql/pg_hba.conf)" > /etc/postgresql/pg_hba.conf cp "$POSTGRES_CONFIG_PATH" "$TMP_CONFIG" # Check if max_slot_wal_keep_size exists in the config - if grep -q "max_slot_wal_keep_size" "$TMP_CONFIG"; then - # Find and replace the existing setting - sed -i 's/^\s*max_slot_wal_keep_size\s*=.*$/max_slot_wal_keep_size = -1/' "$TMP_CONFIG" - else # Add the setting if not found - echo "max_slot_wal_keep_size = -1" >> "$TMP_CONFIG" - fi + echo "max_slot_wal_keep_size = -1" >> "$TMP_CONFIG" # Remove db_user_namespace if upgrading from PG15 if [[ "$OLD_PGVERSION" =~ ^15.* && "$PGVERSION" =~ ^17.* ]]; then @@ -479,13 +474,6 @@ EOF sleep 3 systemctl stop postgresql - # Additional check to ensure postgres is really stopped - if [ -f "${PGDATAOLD}/postmaster.pid" ]; then - echo "PostgreSQL still running, forcing stop..." - pid=$(head -n 1 "${PGDATAOLD}/postmaster.pid") - kill -9 "$pid" || true - rm -f "${PGDATAOLD}/postmaster.pid" - fi else CI_stop_postgres fi From f474c9966db7acee9cdf0c4a8069a1b99b22ca6b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 15 May 2025 13:12:31 -0400 Subject: [PATCH 22/22] chore: version bump --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index c6d15625e..50c6662ce 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.079-orioledb-pgup-1" - postgres17: "17.4.1.029-pgup-1" - postgres15: "15.8.1.086-pgup-1" + postgresorioledb-17: "17.0.1.080-orioledb" + postgres17: "17.4.1.030" + postgres15: "15.8.1.087" # Non Postgres Extensions pgbouncer_release: "1.19.0"