diff --git a/.github/workflows/ami-release-nix-single.yml b/.github/workflows/ami-release-nix-single.yml index 863135ef3..0534a98d0 100644 --- a/.github/workflows/ami-release-nix-single.yml +++ b/.github/workflows/ami-release-nix-single.yml @@ -19,7 +19,7 @@ permissions: jobs: build: - runs-on: arm-runner + runs-on: large-linux-arm timeout-minutes: 150 steps: @@ -40,7 +40,7 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | - PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ env.POSTGRES_MAJOR_VERSION }}'"]' ansible/vars.yml) + PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ env.POSTGRES_MAJOR_VERSION }}'"]' ansible/vars.yml) PG_VERSION=$(echo "$PG_VERSION" | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl # Ensure there's a newline at the end of the file diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index ee78de99c..449903484 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -17,7 +17,7 @@ permissions: jobs: prepare: - runs-on: ubuntu-latest + runs-on: large-linux-x86 outputs: postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} steps: @@ -38,11 +38,7 @@ jobs: matrix: postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} include: - - runner: arm-runner - arch: arm64 - ubuntu_release: focal - ubuntu_version: 20.04 - mcpu: neoverse-n1 + - runner: large-linux-arm runs-on: ${{ matrix.runner }} timeout-minutes: 150 @@ -55,7 +51,7 @@ jobs: - name: Run checks if triggered manually if: ${{ github.event_name == 'workflow_dispatch' }} run: | - SUFFIX=$(sudo nix run nixpkgs#yq -- ".postgres_release[\"postgres${{ matrix.postgres_version }}\"]" ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') + SUFFIX=$(nix run nixpkgs#yq -- ".postgres_release[\"postgres${{ matrix.postgres_version }}\"]" ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') if [[ -z "$SUFFIX" ]] ; then echo "Version must include non-numeric characters if built manually." exit 1 @@ -66,7 +62,7 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | - PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo "$PG_VERSION" | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl # Ensure there's a newline at the end of the file diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 1ac5b2ccc..263b0112a 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -16,7 +16,7 @@ permissions: jobs: prepare: - runs-on: ubuntu-latest + runs-on: large-linux-x86 outputs: matrix_config: ${{ steps.set-matrix.outputs.matrix_config }} steps: @@ -49,7 +49,7 @@ jobs: needs: prepare strategy: matrix: ${{ fromJson(needs.prepare.outputs.matrix_config) }} - runs-on: ubuntu-latest + runs-on: large-linux-x86 outputs: build_args: ${{ steps.args.outputs.result }} steps: @@ -75,7 +75,7 @@ jobs: matrix: postgres: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} arch: [amd64, arm64] - runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-latest' || 'arm-runner' }} + runs-on: ${{ matrix.arch == 'amd64' && 'large-linux-x86' || 'large-linux-arm' }} timeout-minutes: 180 steps: - name: Checkout Repo @@ -93,7 +93,7 @@ jobs: id: image run: | if [[ "${{ matrix.arch }}" == "arm64" ]]; then - pg_version=$(sudo nix run nixpkgs#nushell -- -c ' + pg_version=$(nix run nixpkgs#nushell -- -c ' let version = "${{ matrix.postgres.version }}" let release_key = if ($version | str contains "orioledb") { $"postgresorioledb-17" @@ -132,7 +132,7 @@ jobs: strategy: matrix: include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} - runs-on: ubuntu-latest + runs-on: large-linux-x86 steps: - name: Checkout Repo uses: supabase/postgres/.github/actions/shared-checkout@HEAD @@ -176,7 +176,7 @@ jobs: ${{ steps.get_version.outputs.pg_version }}_arm64 combine_results: needs: [prepare, merge_manifest] - runs-on: ubuntu-latest + runs-on: large-linux-x86 steps: - name: Checkout Repo uses: supabase/postgres/.github/actions/shared-checkout@HEAD diff --git a/.github/workflows/manual-docker-release.yml b/.github/workflows/manual-docker-release.yml index e702014dc..3f86643ff 100644 --- a/.github/workflows/manual-docker-release.yml +++ b/.github/workflows/manual-docker-release.yml @@ -13,7 +13,7 @@ permissions: jobs: prepare: - runs-on: ubuntu-latest + runs-on: large-linux-x86 outputs: matrix_config: ${{ steps.set-matrix.outputs.matrix_config }} steps: @@ -46,7 +46,7 @@ jobs: needs: prepare strategy: matrix: ${{ fromJson(needs.prepare.outputs.matrix_config) }} - runs-on: ubuntu-latest + runs-on: large-linux-x86 outputs: build_args: ${{ steps.args.outputs.result }} steps: @@ -72,7 +72,7 @@ jobs: matrix: postgres: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} arch: [amd64, arm64] - runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-latest' || 'arm-runner' }} + runs-on: ${{ matrix.arch == 'amd64' && 'large-linux-x86' || 'large-linux-arm' }} timeout-minutes: 180 steps: - name: Checkout Repo @@ -90,7 +90,7 @@ jobs: id: image run: | if [[ "${{ matrix.arch }}" == "arm64" ]]; then - pg_version=$(sudo nix run nixpkgs#nushell -- -c ' + pg_version=$(nix run nixpkgs#nushell -- -c ' let version = "${{ matrix.postgres.version }}" let release_key = if ($version | str contains "orioledb") { $"postgresorioledb-17" @@ -141,7 +141,7 @@ jobs: strategy: matrix: include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} - runs-on: ubuntu-latest + runs-on: large-linux-x86 steps: - name: Checkout Repo uses: supabase/postgres/.github/actions/shared-checkout@HEAD @@ -185,7 +185,7 @@ jobs: ${{ steps.get_version.outputs.pg_version }}_arm64 combine_results: needs: [prepare, merge_manifest] - runs-on: ubuntu-latest + runs-on: large-linux-x86 steps: - name: Checkout Repo uses: supabase/postgres/.github/actions/shared-checkout@HEAD diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index f641b7306..82aab37f7 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -10,8 +10,8 @@ on: permissions: contents: read - id-token: write - + id-token: write + jobs: build-run-image: strategy: @@ -37,21 +37,27 @@ jobs: aws-region: "us-east-1" output-credentials: true role-duration-seconds: 7200 + - name: Setup AWS credentials for Nix + if: ${{ github.secret_source == 'Actions' }} + run: | + sudo -H aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + sudo -H aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + sudo -H aws configure set aws_session_token $AWS_SESSION_TOKEN - name: write secret key # use python so we don't interpolate the secret into the workflow logs, in case of bugs run: | - python -c "import os; file = open('nix-secret-key', 'w'); file.write(os.environ['NIX_SIGN_SECRET_KEY']); file.close()" + sudo mkdir -p /etc/nix + sudo -E python -c "import os; file = open('/etc/nix/nix-secret-key', 'w'); file.write(os.environ['NIX_SIGN_SECRET_KEY']); file.close()" env: NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - name: Setup cache script if: ${{ github.secret_source == 'Actions' }} run: | - sudo mkdir -p /etc/nix cat << 'EOF' | sudo tee /etc/nix/upload-to-cache.sh > /dev/null #!/usr/bin/env bash set -eouf export IFS=' ' - echo /nix/var/nix/profiles/default/bin/nix copy --to 's3://nix-postgres-artifacts?secret-key=nix-secret-key' \$OUT_PATHS + /nix/var/nix/profiles/default/bin/nix copy --to 's3://nix-postgres-artifacts?secret-key=/etc/nix/nix-secret-key' $OUT_PATHS EOF sudo chmod +x /etc/nix/upload-to-cache.sh - name: Install nix diff --git a/.github/workflows/publish-migrations-prod.yml b/.github/workflows/publish-migrations-prod.yml index e3d7365cb..ffb633683 100644 --- a/.github/workflows/publish-migrations-prod.yml +++ b/.github/workflows/publish-migrations-prod.yml @@ -5,7 +5,7 @@ on: jobs: build: - runs-on: [self-hosted, linux] + runs-on: large-linux-arm timeout-minutes: 15 permissions: id-token: write diff --git a/.github/workflows/publish-migrations-staging.yml b/.github/workflows/publish-migrations-staging.yml index 9cef8be4b..7acb46d22 100644 --- a/.github/workflows/publish-migrations-staging.yml +++ b/.github/workflows/publish-migrations-staging.yml @@ -8,7 +8,7 @@ on: jobs: build: - runs-on: [self-hosted, linux] + runs-on: large-linux-arm timeout-minutes: 15 permissions: id-token: write diff --git a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml index 847e6d47c..f816415b3 100644 --- a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml +++ b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml @@ -12,7 +12,7 @@ permissions: jobs: prepare: - runs-on: ubuntu-latest + runs-on: large-linux-x86 outputs: postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} steps: @@ -29,7 +29,7 @@ jobs: publish-staging: needs: prepare - runs-on: ubuntu-latest + runs-on: large-linux-x86 strategy: matrix: postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} @@ -79,7 +79,7 @@ jobs: SLACK_FOOTER: '' publish-prod: - runs-on: ubuntu-latest + runs-on: large-linux-x86 if: github.ref_name == 'develop' || contains( github.ref, 'release' ) needs: prepare strategy: @@ -93,7 +93,7 @@ jobs: - name: Grab release version id: process_release_version run: | - VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) VERSION=$(echo $VERSION | tr -d '"') # Remove any surrounding quotes echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo "major_version=$(echo $VERSION | cut -d'.' -f1)" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/publish-nix-pgupgrade-scripts.yml b/.github/workflows/publish-nix-pgupgrade-scripts.yml index 19e255fca..c58e90d83 100644 --- a/.github/workflows/publish-nix-pgupgrade-scripts.yml +++ b/.github/workflows/publish-nix-pgupgrade-scripts.yml @@ -19,7 +19,7 @@ permissions: jobs: prepare: - runs-on: ubuntu-latest + runs-on: large-linux-x86 outputs: postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} steps: @@ -36,7 +36,7 @@ jobs: publish-staging: needs: prepare - runs-on: ubuntu-latest + runs-on: large-linux-x86 strategy: matrix: postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} @@ -84,7 +84,7 @@ jobs: SLACK_FOOTER: '' publish-prod: needs: prepare - runs-on: ubuntu-latest + runs-on: large-linux-x86 if: github.ref_name == 'develop' || contains( github.ref, 'release' ) strategy: diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 8abee1461..35b7a583a 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -18,7 +18,7 @@ permissions: jobs: prepare: - runs-on: ubuntu-latest + runs-on: large-linux-x86 outputs: postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} steps: @@ -38,7 +38,7 @@ jobs: strategy: matrix: postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} - runs-on: arm-native-runner + runs-on: large-linux-arm timeout-minutes: 150 permissions: contents: write @@ -54,7 +54,7 @@ jobs: - name: Run checks if triggered manually if: ${{ github.event_name == 'workflow_dispatch' }} run: | - SUFFIX=$(sudo nix run nixpkgs#yq -- ".postgres_release[\"postgres${{ matrix.postgres_version }}\"]" ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') + SUFFIX=$(nix run nixpkgs#yq -- ".postgres_release[\"postgres${{ matrix.postgres_version }}\"]" ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') if [[ -z $SUFFIX ]] ; then echo "Version must include non-numeric characters if built manually." exit 1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ac285af26..b907c85d7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,9 +1,10 @@ name: Test Database on: - push: - branches: - - develop - pull_request: + # Trigger this workflow when the "Nix CI" workflow completes + workflow_run: + workflows: ["Nix CI"] + types: + - completed workflow_dispatch: permissions: @@ -12,17 +13,13 @@ permissions: jobs: prepare: - runs-on: ubuntu-latest + if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }} + runs-on: large-linux-x86 outputs: postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} steps: - name: Checkout Repo uses: supabase/postgres/.github/actions/shared-checkout@HEAD - - - name: Clear Nix cache - run: | - sudo rm -rf /home/runner/.cache/nix - - uses: DeterminateSystems/nix-installer-action@main with: extra-conf: | @@ -34,14 +31,15 @@ jobs: VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c "split(\"\n\")[:-1]") echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT build: + if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }} needs: prepare strategy: matrix: postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} include: - - runner: ubuntu-22.04 + - runner: large-linux-x86 arch: amd64 - - runner: ubuntu-22.04 + - runner: large-linux-arm arch: arm64 runs-on: ${{ matrix.runner }} timeout-minutes: 180 @@ -51,9 +49,6 @@ jobs: steps: - name: Checkout Repo uses: supabase/postgres/.github/actions/shared-checkout@HEAD - - name: Clear Nix cache - run: | - sudo rm -rf /home/runner/.cache/nix - uses: DeterminateSystems/nix-installer-action@main with: extra-conf: | diff --git a/.github/workflows/testinfra-ami-build.yml b/.github/workflows/testinfra-ami-build.yml index 752bc3938..2e1edda45 100644 --- a/.github/workflows/testinfra-ami-build.yml +++ b/.github/workflows/testinfra-ami-build.yml @@ -1,7 +1,11 @@ name: Testinfra Integration Tests Nix on: - pull_request: + # Trigger this workflow when the "Nix CI" workflow completes + workflow_run: + workflows: ["Nix CI"] + types: + - completed workflow_dispatch: permissions: @@ -10,20 +14,15 @@ permissions: jobs: prepare: - runs-on: ubuntu-latest + if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }} + runs-on: large-linux-x86 outputs: postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} steps: - name: Checkout Repo uses: supabase/postgres/.github/actions/shared-checkout@HEAD - - - uses: DeterminateSystems/nix-installer-action@main - - name: Clean Nix store before build - run: | - sudo nix-collect-garbage -d || true - sudo nix-store --optimize || true - df -h / # Display available space + - uses: DeterminateSystems/nix-installer-action@main - name: Set PostgreSQL versions id: set-versions @@ -32,6 +31,7 @@ jobs: echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT test-ami-nix: + if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }} needs: prepare strategy: fail-fast: false @@ -71,7 +71,6 @@ jobs: - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV - - name: Generate common-nix.vars.pkr.hcl run: | @@ -87,26 +86,12 @@ jobs: GIT_SHA=${{github.sha}} packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - - name: Clean up after AMI stage 1 - if: always() # Run even if previous steps fail - run: | - sudo nix-collect-garbage -d # Delete old generations of all profiles - sudo rm -rf /tmp/* # Clean temporary files - df -h / # Display available space - - name: Build AMI stage 2 run: | packer init stage2-nix-psql.pkr.hcl GIT_SHA=${{github.sha}} packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "git_sha=${GITHUB_SHA}" stage2-nix-psql.pkr.hcl - - name: Clean up after AMI stage 2 - if: always() # Run even if previous steps fail - run: | - sudo nix-collect-garbage -d # Delete old generations of all profiles - sudo rm -rf /tmp/* # Clean temporary files - df -h / # Display available space - - name: Run tests timeout-minutes: 10 env: diff --git a/flake.nix b/flake.nix index a7f48d58c..335ccdf42 100644 --- a/flake.nix +++ b/flake.nix @@ -356,7 +356,6 @@ { inherit (paths) migrationsDir postgresqlSchemaSql pgbouncerAuthSchemaSql statExtensionSql; } '' - set -x mkdir -p $out/bin $out/etc/postgresql-custom $out/etc/postgresql $out/extension-custom-scripts # Copy config files with error handling @@ -1380,7 +1379,7 @@ psql_15 = makeCheckHarness basePackages.psql_15.bin; psql_17 = makeCheckHarness basePackages.psql_17.bin; psql_orioledb-17 = makeCheckHarness basePackages.psql_orioledb-17.bin; - inherit (basePackages) wal-g-2 wal-g-3; + inherit (basePackages) wal-g-2 wal-g-3 dbmate-tool pg_regress; } // pkgs.lib.optionalAttrs (system == "aarch64-linux") { inherit (basePackages) postgresql_15_debug postgresql_15_src postgresql_orioledb-17_debug postgresql_orioledb-17_src postgresql_17_debug postgresql_17_src; }; @@ -1456,6 +1455,7 @@ dbmate nushell pythonEnv + nix-fast-build ]; shellHook = '' export HISTFILE=.history