parity coverage automation

.github/bot_templates/PARITY_COVERAGE_DOCS_PR.md

@@ -0,0 +1,18 @@
+# 📖 Parity Metrics Docs Update Report 📖
+This PR has been automatically generated to update the AWS parity coverage docs.
+It aggregates the latest parity coverage test results from our [test pipeline on GitHub](https://github.com/localstack/localstack/actions/workflows/aws-main.yml?query=branch%3Amaster) as well as from our Pro integration tests.
+
+## 👷🏽 Handle this PR
+The following options describe how to interact with this PR / the auto-update:
+
+✔️ **Accept Changes**
+If the changes are satisfying, just squash-merge the PR and delete the source branch.
+
+🚫 **Ignore Changes**
+If you want to ignore the changes in this PR, just close the PR and *do not delete* the source branch. The PR will not be opened and a new PR will not be created for as long as the generated code does not change (or the branch is deleted). As soon as there are new changes, a new PR will be created.
+
+✏️ **Adapt Changes**
+*Don't do this.* The APIs are auto-generated. If you decide that the APIs should look different, you have to change the code-generation.
+
+⏸️ **Pause Updates**
+Remove the cron-schedule trigger of the GitHub Action workflow which creates these PRs. The action can then still be triggered manually, but it will not be executed automatically.

.github/workflows/docs-parity-updates.yml

@@ -0,0 +1,141 @@
+name: Update Parity Docs
+
+on:
+  schedule:
+    - cron: 0 5 * * MON
+  workflow_dispatch:
+    inputs:
+        targetBranch:
+          required: false
+          type: string
+          default: 'master'
+
+jobs:
+  update-parity-coverage-docs:
+    name: Update Parity Docs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout docs
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          path: docs
+          ref: ${{ github.event.inputs.targetBranch || 'master' }}
+
+      - name: Set up system wide dependencies
+        run: |
+          sudo apt-get install jq wget
+
+      - name: Set up Python 3.11
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Download scripts from meta repository
+        run: |
+          curl -o /tmp/get_latest_github_metrics.sh -L https://raw.githubusercontent.com/localstack/meta/main/scripts/get_latest_github_metrics.sh -H 'Accept: application/vnd.github.v3.raw'
+          chmod +x /tmp/get_latest_github_metrics.sh
+
+      - name: Download metrics data from Moto Integration test pipeline (GitHub)
+        working-directory: docs
+        run: /tmp/get_latest_github_metrics.sh ./target main
+        env:
+          GITHUB_TOKEN: ${{ secrets.PRO_ACCESS_TOKEN }}
+          REPOSITORY_NAME: localstack-moto-test-coverage 
+          ARTIFACT_ID: test-metrics
+          WORKFLOW: moto-integration-tests
+          PREFIX_ARTIFACT: moto-integration-test
+          FILTER_SUCCESS: 0
+
+      - name: Download metrics data from Terraform Integration test pipeline (GitHub)
+        working-directory: docs
+        run: /tmp/get_latest_github_metrics.sh ./target main
+        env:
+          GITHUB_TOKEN: ${{ secrets.PRO_ACCESS_TOKEN }}
+          REPOSITORY_NAME: localstack-terraform-test 
+          ARTIFACT_ID: test-metrics
+          WORKFLOW: "Terraform Tests"
+          FILTER_SUCCESS: 0
+
+      - name: Download metrics data from Pro pipeline (GitHub)
+        working-directory: docs
+        run: /tmp/get_latest_github_metrics.sh ./target master
+        env:
+          GITHUB_TOKEN: ${{ secrets.PRO_ACCESS_TOKEN }}
+          REPOSITORY_NAME: localstack-ext
+          ARTIFACT_ID: parity-metric-ext-raw-*
+          WORKFLOW: "AWS / Build, Test, Push" 
+          PREFIX_ARTIFACT: pro-integration-test
+
+      - name: Download coverage (capture-notimplemented) data from Pro pipeline (GitHub)
+        working-directory: docs
+        run: /tmp/get_latest_github_metrics.sh ./target master
+        env:
+          GITHUB_TOKEN: ${{ secrets.PRO_ACCESS_TOKEN }}
+          REPOSITORY_NAME: localstack-ext
+          ARTIFACT_ID: capture-notimplemented-pro 
+          WORKFLOW: "AWS / Build, Test, Push" 
+          RESOURCE_FOLDER: "metrics-implementation-details"
+
+      - name: Download metrics data from Community pipeline (GitHub)
+        working-directory: docs
+        run: /tmp/get_latest_github_metrics.sh ./target master
+        env:
+          GITHUB_TOKEN: ${{ secrets.PRO_ACCESS_TOKEN }}
+          REPOSITORY_NAME: localstack
+          ARTIFACT_ID: parity-metric-raw-amd*
+          WORKFLOW: "AWS / Build, Test, Push"
+          PREFIX_ARTIFACT: community-integration-test
+
+      - name: Download coverage (capture-notimplemented) data from Community pipeline (GitHub)
+        working-directory: docs
+        run: /tmp/get_latest_github_metrics.sh ./target master
+        env:
+          GITHUB_TOKEN: ${{ secrets.PRO_ACCESS_TOKEN }}
+          REPOSITORY_NAME: localstack
+          ARTIFACT_ID: capture-notimplemented
+          WORKFLOW: "AWS / Build, Test, Push"
+          RESOURCE_FOLDER: "metrics-implementation-details/community"
+
+      - name: Create Parity Coverage Docs
+        working-directory: docs
+        run: |
+          python3 -m scripts.create_data_coverage -i target/metrics-implementation-details -r target/metrics-raw -o target/updated_coverage -s src/data/coverage/service_display_name.json
+          mv -f target/updated_coverage/data/*.json src/data/coverage
+
+      - name: Check for changes
+        id: check-for-changes
+        working-directory: docs
+        run: |
+          # Check if there are changed files and store the result in resources/diff-check.log
+          # Check against the PR branch if it exists, otherwise against the master
+          # Store the result in resources/diff-check.log and store the diff count in the GitHub Action output "diff-count"
+          mkdir -p resources
+          (git diff --name-only origin/parity-coverage-auto-updates data/coverage/ 2>/dev/null || git diff --name-only origin/${{ github.event.inputs.targetBranch || 'master' }} src/data/coverage/ 2>/dev/null) | tee -a resources/diff-check.log
+          echo "diff-count=$(cat resources/diff-check.log | wc -l)" >> $GITHUB_OUTPUT
+
+      - name: Read PR markdown template
+        if: ${{ success() && steps.check-for-changes.outputs.diff-count != '0' && steps.check-for-changes.outputs.diff-count != '' }}
+        id: template
+        uses: juliangruber/read-file-action@v1
+        with:
+          path: docs/.github/bot_templates/PARITY_COVERAGE_DOCS_PR.md
+
+      - name: Create PR
+        uses: peter-evans/create-pull-request@v7
+        if: ${{ success() && steps.check-for-changes.outputs.diff-count != '0' && steps.check-for-changes.outputs.diff-count != '' }}
+        with:
+          path: docs
+          title: "Update Parity Coverage Docs"
+          body: "${{ steps.template.outputs.content }}"
+          branch: "parity-coverage-auto-updates"
+          author: "LocalStack Bot <[email protected]>"
+          committer: "LocalStack Bot <[email protected]>"
+          commit-message: "update generated parity coverage docs"
+          token: ${{ secrets.PRO_ACCESS_TOKEN }}

.gitignore

@@ -33,3 +33,6 @@ __pycache__/
 
 # diff check
 resources/diff-check.log
+
+# Parity coverage
+target/