diff --git a/.github/workflows/ami-release-nix-single.yml b/.github/workflows/ami-release-nix-single.yml index 86a1d40fc..17c6cfb06 100644 --- a/.github/workflows/ami-release-nix-single.yml +++ b/.github/workflows/ami-release-nix-single.yml @@ -18,15 +18,20 @@ permissions: id-token: write jobs: + checkout: + uses: ./.github/workflows/shared-checkout.yml + build: + needs: checkout runs-on: arm-runner timeout-minutes: 150 steps: - - name: Checkout Repo - uses: actions/checkout@v3 + - name: Download repository + uses: actions/download-artifact@v4 with: - ref: ${{ github.event.inputs.branch }} + name: repository + path: . - name: Get current branch SHA id: get_sha diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 004a597d7..99873aa35 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -16,14 +16,21 @@ permissions: id-token: write jobs: + checkout: + uses: ./.github/workflows/shared-checkout.yml + prepare: + needs: checkout runs-on: ubuntu-latest outputs: postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} steps: - - name: Checkout Repo - uses: actions/checkout@v3 - + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . + - uses: DeterminateSystems/nix-installer-action@main - name: Set PostgreSQL versions @@ -33,7 +40,7 @@ jobs: echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT build: - needs: prepare + needs: [checkout, prepare] strategy: matrix: postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} @@ -47,8 +54,11 @@ jobs: timeout-minutes: 150 steps: - - name: Checkout Repo - uses: actions/checkout@v3 + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . - uses: DeterminateSystems/nix-installer-action@main diff --git a/.github/workflows/check-shellscripts.yml b/.github/workflows/check-shellscripts.yml index ab0f59b87..b54a264e4 100644 --- a/.github/workflows/check-shellscripts.yml +++ b/.github/workflows/check-shellscripts.yml @@ -11,20 +11,29 @@ permissions: contents: read jobs: + checkout: + uses: ./.github/workflows/shared-checkout.yml + build: + needs: checkout runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Run ShellCheck - uses: ludeeus/action-shellcheck@master - env: - SHELLCHECK_OPTS: -e SC2001 -e SC2002 -e SC2143 - with: - scandir: './ansible/files/admin_api_scripts' + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . + + - name: Run ShellCheck + uses: ludeeus/action-shellcheck@master + env: + SHELLCHECK_OPTS: -e SC2001 -e SC2002 -e SC2143 + with: + scandir: './ansible/files/admin_api_scripts' - - name: Run ShellCheck on pg_upgrade scripts - uses: ludeeus/action-shellcheck@master - env: - SHELLCHECK_OPTS: -e SC2001 -e SC2002 -e SC2143 - with: - scandir: './ansible/files/admin_api_scripts/pg_upgrade_scripts' + - name: Run ShellCheck on pg_upgrade scripts + uses: ludeeus/action-shellcheck@master + env: + SHELLCHECK_OPTS: -e SC2001 -e SC2002 -e SC2143 + with: + scandir: './ansible/files/admin_api_scripts/pg_upgrade_scripts' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93a2afdc5..33fec7345 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,12 +7,19 @@ permissions: contents: read jobs: + checkout: + uses: ./.github/workflows/shared-checkout.yml + check-release-version: + needs: checkout timeout-minutes: 5 runs-on: ubuntu-latest steps: - - name: Checkout Repo - uses: actions/checkout@v3 + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . - name: Load postgres_release values id: load_postgres_release diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index ed7075d1d..3f80354d3 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -15,14 +15,22 @@ permissions: id-token: write jobs: + checkout: + uses: ./.github/workflows/shared-checkout.yml + prepare: + needs: checkout runs-on: ubuntu-latest outputs: matrix_config: ${{ steps.set-matrix.outputs.matrix_config }} steps: + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . + - uses: DeterminateSystems/nix-installer-action@main - - name: Checkout Repo - uses: actions/checkout@v3 - name: Generate build matrix id: set-matrix run: | @@ -53,7 +61,12 @@ jobs: outputs: build_args: ${{ steps.args.outputs.result }} steps: - - uses: actions/checkout@v3 + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . + - uses: DeterminateSystems/nix-installer-action@main - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.version }}" >> $GITHUB_ENV @@ -77,7 +90,12 @@ jobs: runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-latest' || 'arm-runner' }} timeout-minutes: 180 steps: - - uses: actions/checkout@v3 + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . + - uses: DeterminateSystems/nix-installer-action@main - run: docker context create builders - uses: docker/setup-buildx-action@v3 @@ -132,7 +150,6 @@ jobs: include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - uses: DeterminateSystems/nix-installer-action@main - uses: docker/setup-buildx-action@v3 - uses: docker/login-action@v2 @@ -175,7 +192,6 @@ jobs: needs: [prepare, merge_manifest] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - uses: DeterminateSystems/nix-installer-action@main - name: Debug Input from Prepare diff --git a/.github/workflows/manual-docker-release.yml b/.github/workflows/manual-docker-release.yml index 8948324db..a3198770b 100644 --- a/.github/workflows/manual-docker-release.yml +++ b/.github/workflows/manual-docker-release.yml @@ -12,14 +12,22 @@ permissions: contents: read jobs: + checkout: + uses: ./.github/workflows/shared-checkout.yml + prepare: + needs: checkout runs-on: ubuntu-latest outputs: matrix_config: ${{ steps.set-matrix.outputs.matrix_config }} steps: + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . + - uses: DeterminateSystems/nix-installer-action@main - - name: Checkout Repo - uses: actions/checkout@v3 - name: Generate build matrix id: set-matrix run: | @@ -50,7 +58,12 @@ jobs: outputs: build_args: ${{ steps.args.outputs.result }} steps: - - uses: actions/checkout@v3 + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . + - uses: DeterminateSystems/nix-installer-action@main - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.version }}" >> $GITHUB_ENV @@ -74,7 +87,12 @@ jobs: runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-latest' || 'arm-runner' }} timeout-minutes: 180 steps: - - uses: actions/checkout@v3 + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . + - uses: DeterminateSystems/nix-installer-action@main - run: docker context create builders - uses: docker/setup-buildx-action@v3 @@ -141,7 +159,6 @@ jobs: include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - uses: DeterminateSystems/nix-installer-action@main - uses: docker/setup-buildx-action@v3 - uses: docker/login-action@v2 @@ -184,7 +201,6 @@ jobs: needs: [prepare, merge_manifest] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - uses: DeterminateSystems/nix-installer-action@main - name: Debug Input from Prepare diff --git a/.github/workflows/mirror-postgrest.yml b/.github/workflows/mirror-postgrest.yml index 1658730f7..e4f55adb8 100644 --- a/.github/workflows/mirror-postgrest.yml +++ b/.github/workflows/mirror-postgrest.yml @@ -12,12 +12,21 @@ permissions: contents: read jobs: + checkout: + uses: ./.github/workflows/shared-checkout.yml + version: + needs: checkout runs-on: ubuntu-latest outputs: postgrest_release: ${{ steps.args.outputs.result }} steps: - - uses: actions/checkout@v4 + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . + - id: args uses: mikefarah/yq@master with: diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index c1811fd00..8f62cf7ab 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -13,7 +13,11 @@ permissions: id-token: write jobs: + checkout: + uses: ./.github/workflows/shared-checkout.yml + build-run-image: + needs: checkout strategy: fail-fast: false matrix: @@ -27,13 +31,12 @@ jobs: runs-on: ${{ matrix.runner }} timeout-minutes: 180 steps: - - - name: Check out code - uses: actions/checkout@v4 + - name: Download repository + uses: actions/download-artifact@v4 with: - ref: ${{ github.event.pull_request.head.ref || github.ref }} - fetch-depth: 0 - fetch-tags: true + name: repository + path: . + - name: aws-creds uses: aws-actions/configure-aws-credentials@v4 with: diff --git a/.github/workflows/publish-migrations-prod.yml b/.github/workflows/publish-migrations-prod.yml index d7e813667..442f74165 100644 --- a/.github/workflows/publish-migrations-prod.yml +++ b/.github/workflows/publish-migrations-prod.yml @@ -4,7 +4,11 @@ on: workflow_dispatch: jobs: + checkout: + uses: ./.github/workflows/shared-checkout.yml + build: + needs: checkout runs-on: [self-hosted, linux] timeout-minutes: 15 permissions: @@ -21,9 +25,6 @@ jobs: env: GITHUB_REF: ${{ github.ref }} - - name: Checkout Repo - uses: actions/checkout@v2 - - name: Merging migration files run: cat $(ls -1) > ../migration-output.sql working-directory: ${{ github.workspace }}/migrations/db/migrations diff --git a/.github/workflows/publish-migrations-staging.yml b/.github/workflows/publish-migrations-staging.yml index 72b28e927..e033b922e 100644 --- a/.github/workflows/publish-migrations-staging.yml +++ b/.github/workflows/publish-migrations-staging.yml @@ -7,7 +7,11 @@ on: workflow_dispatch: jobs: + checkout: + uses: ./.github/workflows/shared-checkout.yml + build: + needs: checkout runs-on: [self-hosted, linux] timeout-minutes: 15 permissions: @@ -15,9 +19,6 @@ jobs: contents: read steps: - - name: Checkout Repo - uses: actions/checkout@v2 - - name: Merging migration files run: cat $(ls -1) > ../migration-output.sql working-directory: ${{ github.workspace }}/migrations/db/migrations diff --git a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml index bec85166e..37120be88 100644 --- a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml +++ b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml @@ -11,14 +11,21 @@ permissions: id-token: write jobs: + checkout: + uses: ./.github/workflows/shared-checkout.yml + prepare: + needs: checkout runs-on: ubuntu-latest outputs: postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} steps: - - name: Checkout Repo - uses: actions/checkout@v3 - + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . + - uses: DeterminateSystems/nix-installer-action@main - name: Set PostgreSQL versions @@ -33,12 +40,14 @@ jobs: strategy: matrix: postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - - uses: DeterminateSystems/nix-installer-action@main + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . + + - uses: DeterminateSystems/nix-installer-action@main - name: Grab release version id: process_release_version @@ -87,8 +96,13 @@ jobs: postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} steps: - - name: Checkout Repo - uses: actions/checkout@v3 + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . + + - uses: DeterminateSystems/nix-installer-action@main - name: Grab release version id: process_release_version @@ -110,7 +124,7 @@ jobs: with: role-to-assume: ${{ secrets.PROD_AWS_ROLE }} aws-region: "us-east-1" - + - name: Upload pg_upgrade scripts to s3 prod run: | aws s3 cp /tmp/pg_upgrade_bin.tar.gz "s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz" diff --git a/.github/workflows/publish-nix-pgupgrade-scripts.yml b/.github/workflows/publish-nix-pgupgrade-scripts.yml index ece3e5966..bc2d662d5 100644 --- a/.github/workflows/publish-nix-pgupgrade-scripts.yml +++ b/.github/workflows/publish-nix-pgupgrade-scripts.yml @@ -18,14 +18,21 @@ permissions: id-token: write jobs: + checkout: + uses: ./.github/workflows/shared-checkout.yml + prepare: + needs: checkout runs-on: ubuntu-latest outputs: postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} steps: - - name: Checkout Repo - uses: actions/checkout@v3 - + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . + - uses: DeterminateSystems/nix-installer-action@main - name: Set PostgreSQL versions @@ -42,8 +49,11 @@ jobs: postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} steps: - - name: Checkout Repo - uses: actions/checkout@v3 + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . - uses: DeterminateSystems/nix-installer-action@main @@ -91,11 +101,13 @@ jobs: matrix: postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . + - uses: DeterminateSystems/nix-installer-action@main - name: Grab release version diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 708c7faf2..cab673cb2 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -17,13 +17,20 @@ permissions: id-token: write jobs: + checkout: + uses: ./.github/workflows/shared-checkout.yml + prepare: + needs: checkout runs-on: ubuntu-latest outputs: postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} steps: - - name: Checkout Repo - uses: actions/checkout@v3 + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . - uses: DeterminateSystems/nix-installer-action@main @@ -46,8 +53,11 @@ jobs: id-token: write steps: - - name: Checkout Repo - uses: actions/checkout@v3 + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . - uses: DeterminateSystems/nix-installer-action@main diff --git a/.github/workflows/shared-checkout.yml b/.github/workflows/shared-checkout.yml new file mode 100644 index 000000000..3322b120e --- /dev/null +++ b/.github/workflows/shared-checkout.yml @@ -0,0 +1,28 @@ +name: Shared Checkout + +on: + workflow_call: + # No inputs needed as we're using the default GitHub context + +jobs: + checkout: + runs-on: ubuntu-latest + outputs: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + fetch-tags: true + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload repository + uses: actions/upload-artifact@v4 + with: + name: repository + path: . + retention-days: 1 \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a218ef882..182be1ed1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,13 +11,20 @@ permissions: id-token: write jobs: + checkout: + uses: ./.github/workflows/shared-checkout.yml + prepare: + needs: checkout runs-on: ubuntu-latest outputs: postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} steps: - - name: Checkout Repo - uses: actions/checkout@v4 + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . - name: Clear Nix cache run: | @@ -48,8 +55,14 @@ jobs: env: POSTGRES_PORT: 5478 POSTGRES_PASSWORD: password + steps: - - uses: actions/checkout@v4 + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . + - name: Clear Nix cache run: | sudo rm -rf /home/runner/.cache/nix @@ -81,9 +94,15 @@ jobs: echo "EOF" >> $GITHUB_OUTPUT - name: verify schema.sql is committed run: | + # Save the original schema file + cp migrations/schema-${{ env.PGMAJOR }}.sql migrations/schema-${{ env.PGMAJOR }}.sql.original + + # Run dbmate-tool nix run github:supabase/postgres/${{ github.sha }}#dbmate-tool -- --version ${{ env.PGMAJOR }} --flake-url github:supabase/postgres/${{ github.sha }} - if ! git diff --exit-code --quiet migrations/schema-${{ env.PGMAJOR }}.sql; then + + # Compare the files + if ! cmp -s migrations/schema-${{ env.PGMAJOR }}.sql migrations/schema-${{ env.PGMAJOR }}.sql.original; then echo "Detected changes in schema.sql:" - git diff migrations/schema-${{ env.PGMAJOR }}.sql + diff -u migrations/schema-${{ env.PGMAJOR }}.sql.original migrations/schema-${{ env.PGMAJOR }}.sql exit 1 fi \ No newline at end of file diff --git a/.github/workflows/testinfra-ami-build.yml b/.github/workflows/testinfra-ami-build.yml index 2b07e716f..dc956ec57 100644 --- a/.github/workflows/testinfra-ami-build.yml +++ b/.github/workflows/testinfra-ami-build.yml @@ -9,14 +9,21 @@ permissions: id-token: write jobs: + checkout: + uses: ./.github/workflows/shared-checkout.yml + prepare: + needs: checkout runs-on: ubuntu-latest outputs: postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} steps: - - name: Checkout Repo - uses: actions/checkout@v4 - + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . + - uses: DeterminateSystems/nix-installer-action@main - name: Clean Nix store before build @@ -51,8 +58,19 @@ jobs: id-token: write steps: - - name: Checkout Repo - uses: actions/checkout@v4 + - name: Download repository + uses: actions/download-artifact@v4 + with: + name: repository + path: . + + - name: Set script permissions + run: | + chmod +x ebssurrogate/scripts/chroot-bootstrap-nix.sh + chmod +x ebssurrogate/scripts/surrogate-bootstrap-nix.sh + chmod +x migrations/db/migrate.sh + + - uses: DeterminateSystems/nix-installer-action@main - id: args uses: mikefarah/yq@master