tests.yml

# The goals of this workflow are to check that:
# - we can build semgrep-core and semgrep
# - we can build a Docker image as well as Linux and MacOS binaries
# - all our tests (the one in semgrep-core and the one in semgrep-cli) are passing
# - we don't have any perf regressions in our benchmarks

name: tests

on:
  workflow_dispatch:
  pull_request:
    paths-ignore:
      - "**.md"
  push:
    branches:
      - develop
    paths-ignore:
      - "**.md"

jobs:
  test-core:
    # TODO: why not use the artifact of build-semgrep-core in this job instead?
    name: test semgrep-core
    runs-on: ubuntu-22.04
    container: returntocorp/ocaml:alpine-2023-04-03
    env:
      HOME: /root
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: true
          persist-credentials: false
      - name: Build semgrep-core
        run: ./scripts/install-alpine-semgrep-core
      - name: Test semgrep-core
        run: |
          eval $(opam env)
          START=`date +%s`
          make core-test
          make core-e2etest
          END=`date +%s`
          TEST_RUN_TIME=$((END-START))
          curl --fail -L -X POST "https://dashboard.semgrep.dev/api/metric/semgrep.core.test-run-time-seconds.num" -d "$TEST_RUN_TIME"
      - name: Report Number of Tests Stats
        if: github.ref == 'refs/heads/develop'
        run: ./tests/report_test_metrics.sh

      # TODO: move this to a stable host for more reliable results.
      #
      # It's not clear how to push the stats only when "on the main
      # branch". The GitHub Actions documentation is unhelpful. So we
      # keep things simple and publish the results every time.
      #
      - name: Publish match performance
        run: |
          # This runs a short test suite to track the match performance
          # of semgrep-core over time. The results are pushed to the
          # dashboard at https://dashboard.semgrep.dev/
          #
          opam exec -- make report-perf-matching

  test-osemgrep:
    name: test osemgrep
    runs-on: ubuntu-22.04
    container: returntocorp/ocaml:alpine-2023-04-03
    env:
      HOME: /root
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: true
          persist-credentials: false
      - name: Build semgrep-core
        run: ./scripts/install-alpine-semgrep-core
      - name: Install osemgrep
        run: |
          eval $(opam env)
          make core-install
          cp bin/semgrep-core /usr/bin/ # needed for pipenv install to work below
      - name: Install Python dependencies
        working-directory: cli
        run: pipenv install --dev
      - name: Run pytest for osemgrep known passing tests
        working-directory: cli
        run: |
          make osempass

  test-cli:
    name: test semgrep-cli
    runs-on: ubuntu-22.04
    needs: [build-test-core-x86]
    permissions:
      contents: write
      pull-requests: write
    strategy:
      matrix:
        python: ["3.7", "3.8", "3.9", "3.10", "3.11"]
    steps:
      - uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: Fetch semgrep-cli submodules
        run: git submodule update --init --recursive --recommend-shallow cli/src/semgrep/lang cli/src/semgrep/semgrep_interfaces
      - uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: pipenv
      - run: pip install pipenv==2022.6.7
      - name: Download artifacts
        uses: actions/download-artifact@v3
        with:
          name: ocaml-build-artifacts-release
      - name: Install artifacts
        run: |
          tar xf ocaml-build-artifacts.tgz
          sudo cp ocaml-build-artifacts/bin/* /usr/bin
      - name: Install Python dependencies
        working-directory: cli
        run: pipenv install --dev
      - name: Run pytest
        working-directory: cli
        run: |
          # tests should simulate CI environment iff they need one
          unset CI
          unset "${!GITHUB_@}"

          pipenv run pytest -n auto -vv --snapshot-update --allow-snapshot-deletion
      # because of the fail-fast setting, we expect only the fastest failing job to get to the steps below
      - name: Prepare repo for snapshot commit
        if: failure()
        run: |
          # the commit step that follows will fail to fetch the pfff submodule
          # (perhaps because of the github token's permissions)
          # so we disable recursive fetching
          git config fetch.recurseSubmodules false
      # Because we are not persisting creds in the checkout step, we must create the creds file
      # when it's needed to push up snapshot changes.
      # Note that this is only done after tests are run, so that tests do not have access to the token
      - name: Configure git creds for push
        id: configure-creds
        if: failure() && github.event_name == 'pull_request' && (github.actor != 'dependabot[bot]' && !(github.event.pull_request.head.repo.full_name != github.repository))
        run: |
          echo "machine github.com" >> ~/.netrc
          echo "login ${{ github.repository }}" >> ~/.netrc
          echo "password ${{ secrets.GITHUB_TOKEN }}" >> ~/.netrc
      - name: Commit snapshot updates
        id: snapshot-commit
        if: failure() && github.event_name == 'pull_request' && (github.actor != 'dependabot[bot]' && !(github.event.pull_request.head.repo.full_name != github.repository))
        uses: EndBug/add-and-commit@v9
        with:
          add: cli/tests/e2e/snapshots
          default_author: github_actions
          message: "Update pytest snapshots"
          new_branch: snapshot-updates-${{ github.run_id }}-${{ github.run_attempt }}
      - name: Remove Credentials
        id: remove-creds
        if: failure() && github.event_name == 'pull_request' && (github.actor != 'dependabot[bot]' && !(github.event.pull_request.head.repo.full_name != github.repository))
        run: rm ~/.netrc
      - name: Comment about any snapshot updates
        if: failure() && steps.snapshot-commit.outputs.pushed == 'true'
        run: |
          echo ":camera_flash: The pytest shapshots changed in your PR." >> /tmp/message.txt
          echo "Please carefully review these changes and make sure they are intended:" >> /tmp/message.txt
          echo >> /tmp/message.txt
          echo "1. Review the changes at https://github.com/returntocorp/semgrep/commit/${{ steps.snapshot-commit.outputs.commit_long_sha }}" >> /tmp/message.txt
          echo "2. Accept the new snapshots with" >> /tmp/message.txt
          echo >> /tmp/message.txt
          echo "       git fetch origin && git cherry-pick ${{ steps.snapshot-commit.outputs.commit_sha }} && git push" >> /tmp/message.txt

          gh pr comment ${{ github.event.pull_request.number }} --body-file /tmp/message.txt
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  # These tests aren't run by default by pytest.
  # To reproduce errors locally, use:
  #   $ cd cli/tests
  #   $ make qa
  #
  # TODO: if you know this, please explain what the code below is meant
  #       to achieve and how to make sure it works.
  #
  test-qa:
    name: quality assurance on semgrep
    runs-on: ubuntu-22.04
    needs: [build-test-core-x86]
    strategy:
      fail-fast: false
      matrix:
        split: [1, 2, 3, 4]
    steps:
      - uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: Fetch semgrep-cli submodules
        run: git submodule update --init --recursive --recommend-shallow cli/src/semgrep/lang cli/src/semgrep/semgrep_interfaces tests/semgrep-rules
      - uses: actions/setup-python@v4
        with:
          python-version: "3.11"
          cache: pipenv
      - run: pip install pipenv==2022.6.7
      - name: Download artifacts
        uses: actions/download-artifact@v3
        with:
          name: ocaml-build-artifacts-release
      - name: Install artifacts
        run: |
          tar xf ocaml-build-artifacts.tgz
          sudo cp ocaml-build-artifacts/bin/* /usr/bin
      - name: Install semgrep
        working-directory: cli
        run: |
          export PATH=/github/home/.local/bin:$PATH
          pipenv install --dev
      - uses: actions/cache@v3
        with:
          path: ~/.cache/qa-public-repos
          key: qa-public-repos-${{ hashFiles('semgrep/tests/qa/*public_repos*') }}-${{ matrix.split }}
      - run: |
          mkdir -p ~/.cache/qa-public-repos
          touch ~/.cache/qa-public-repos/ok
      - name: Test semgrep
        working-directory: cli
        run: |
          export PATH=/github/home/.local/bin:$PATH
          pipenv run pytest -n auto -vv --tb=short --splits 4 --group ${{ matrix.split }} tests/qa
        env:
          QA_TESTS_CACHE_PATH: ~/.cache/qa-public-repos

  # Run abbreviated version of benchmarks to check that they work
  benchmarks-lite:
    runs-on: ubuntu-22.04
    needs: [build-test-core-x86]
    steps:
      - uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: Fetch semgrep-cli submodules
        run: git submodule update --init --recursive --recommend-shallow cli/src/semgrep/lang cli/src/semgrep/semgrep_interfaces
      - uses: actions/setup-python@v4
        with:
          python-version: "3.7"
          cache: pipenv
      - run: pip install pipenv==2022.6.7
      - name: Download artifacts
        uses: actions/download-artifact@v3
        with:
          name: ocaml-build-artifacts-release
      - name: Install artifacts
        run: |
          tar xf ocaml-build-artifacts.tgz
          sudo cp ocaml-build-artifacts/bin/* /usr/bin
      - name: Install cli dependencies
        working-directory: cli
        run: pipenv install --dev
      - name: Test dummy benchmarks on latest
        working-directory: cli
        run: |
          pipenv run semgrep --version
          pipenv run python -m semgrep --version
          pipenv run semgrep-core -version
          pipenv run python3 ../perf/run-benchmarks --dummy

  # Run each benchmark twice to decrease effect of natural variance
  benchmarks-full:
    runs-on: ubuntu-22.04
    needs: [build-test-core-x86]
    steps:
      - uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: Fetch semgrep-cli submodules
        run: git submodule update --init --recursive --recommend-shallow cli/src/semgrep/lang cli/src/semgrep/semgrep_interfaces
      - uses: actions/setup-python@v4
        with:
          python-version: "3.7"
          cache: pipenv
      - run: pip install pipenv==2022.6.7
      - name: Download artifacts
        uses: actions/download-artifact@v3
        with:
          name: ocaml-build-artifacts-release
      - name: Install artifacts
        run: |
          tar xf ocaml-build-artifacts.tgz
          sudo cp ocaml-build-artifacts/bin/* /usr/bin
      - name: Install cli dependencies
        working-directory: cli
        run: pipenv install --dev
      - name: Run perf benchmark
        run: scripts/run-benchmarks.sh ${{ secrets.GITHUB_TOKEN }} ${{ github.event.number }}
      - name: Run python performance tests
        working-directory: cli
        run: pipenv run pytest tests/performance

  build-test-docker:
    uses: ./.github/workflows/build-test-docker.yaml
    secrets: inherit
    with:
      docker-tags: |
        type=semver,pattern={{version}}
        type=semver,pattern={{major}}.{{minor}}
        type=ref,event=pr
        type=ref,event=branch
        type=sha,event=branch
        type=edge
      artifact-name: image-test
      repository-name: ${{ github.repository }}
      file: Dockerfile
      enable-tests: true

  push-docker:
    needs: [build-test-docker]
    uses: ./.github/workflows/push-docker.yaml
    if: github.ref == 'refs/heads/develop' || (github.actor != 'dependabot[bot]' && !(github.event.pull_request.head.repo.full_name != github.repository))
    secrets: inherit
    with:
      artifact-name: image-test
      repository-name: ${{ github.repository }}
      dry-run: false

  build-test-dev-docker:
    uses: ./.github/workflows/build-test-docker.yaml
    secrets: inherit
    with:
      docker-tags: |
        type=semver,pattern={{version}}
        type=semver,pattern={{major}}.{{minor}}
        type=ref,event=pr
        type=ref,event=branch
        type=sha,event=branch
        type=edge
      repository-name: ${{ github.repository }}-dev
      artifact-name: image-dev
      file: dockerfiles/semgrep-dev.Dockerfile
      enable-tests: false

  push-dev-docker:
    needs: [build-test-dev-docker]
    uses: ./.github/workflows/push-docker.yaml
    if: github.ref == 'refs/heads/develop'
    secrets: inherit
    with:
      artifact-name: image-dev
      repository-name: ${{ github.repository }}-dev
      dry-run: false

  test-semgrep-pro:
    needs: [build-test-docker, push-docker]
    uses: ./.github/workflows/test-semgrep-pro.yaml
    if: github.ref == 'refs/heads/develop' || github.event.pull_request.head.repo.full_name == github.repository # only returntocorp has the necessary credentials to access semgrep pro
    secrets: inherit
    with:
      artifact-name: image-test
      repository-name: ${{ github.repository }}

  build-test-core-x86:
    uses: ./.github/workflows/build-test-core-x86.yaml
    secrets: inherit

  build-test-manylinux-x86:
    needs: [build-test-core-x86]
    uses: ./.github/workflows/build-test-manylinux-x86.yaml
    secrets: inherit

  build-test-osx-x86:
    uses: ./.github/workflows/build-test-osx-x86.yaml
    secrets: inherit

  build-test-osx-m1:
    uses: ./.github/workflows/build-test-osx-m1.yaml
    secrets: inherit

  build-test-ubuntu-16-04:
    needs: [build-test-core-x86]
    uses: ./.github/workflows/build-test-ubuntu-x86.yaml
    secrets: inherit

  build-test-javascript:
    uses: ./.github/workflows/build-test-javascript.yaml
    secrets: inherit