Speed up CI: Skip a few steps (rerun-io#5901)

* Part of rerun-io#3033 ### What For PRs, skip: * Building a notebook (moved to `nightly`) * `cargo test` * `roundtrips.py` * `compare_snippet_output.py` ### Effect Total duration: 17m59s -> 12m39s Billable time: 1h15m -> 1h9m ### Checklist * [x] A box was cheked - [PR Build Summary](https://build.rerun.io/pr/5901) - [Recent benchmark results](https://build.rerun.io/graphs/crates.html) - [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)
karmueo · Apr 11, 2024 · 60cf846 · 60cf846
1 parent 13a2193
commit 60cf846
Show file tree

Hide file tree

Showing 16 changed files with 316 additions and 65 deletions.
diff --git a/.github/workflows/contrib_checks.yml b/.github/workflows/contrib_checks.yml
@@ -102,8 +102,6 @@ jobs:
   rs-lints:
     name: Rust lints (fmt, check, cranky, tests, doc)
     runs-on: ubuntu-latest-16-cores
-    container:
-      image: rerunio/ci_docker:0.12.0
     steps:
       - uses: actions/checkout@v4
 
@@ -208,12 +206,6 @@ jobs:
         with:
           python-version: "3.8"
 
-      - name: Install dependencies
-        shell: bash
-        run: |
-          pip install gitignore_parser
-          pip install -r ./scripts/ci/requirements.txt
-
       - name: Rerun lints
         run: pixi run lint-rerun
 
@@ -223,17 +215,17 @@ jobs:
       - name: Check for too large files
         shell: bash
         run: |
-          ./scripts/ci/check_large_files.sh
+          pixi run ./scripts/ci/check_large_files.sh
 
       - name: Check Python example requirements
         shell: bash
         run: |
-          ./scripts/ci/check_requirements.py
+          pixi run ./scripts/ci/check_requirements.py
 
       - name: Check Python example thumbnails
         shell: bash
         run: |
-          ./scripts/ci/thumbnails.py check
+          pixi run ./scripts/ci/thumbnails.py check
 
   spell-check:
     name: Spell Check
@@ -275,8 +267,6 @@ jobs:
   cpp-tests:
     name: C++ tests
     runs-on: ubuntu-latest
-    container:
-      image: rerunio/ci_docker:0.12.0
     steps:
       - uses: actions/checkout@v4
 

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -28,7 +28,7 @@ jobs:
     uses: ./.github/workflows/reusable_checks_rust.yml
     with:
       CONCURRENCY: nightly
-      ALL_CHECKS: true
+      CHANNEL: nightly
     secrets: inherit
 
   checks-python:
@@ -39,7 +39,7 @@ jobs:
     secrets: inherit
 
   build-web:
-    name: "Build and upload web viewer"
+    name: "Build web viewer"
     uses: ./.github/workflows/reusable_build_web.yml
     with:
       CONCURRENCY: nightly
@@ -65,6 +65,15 @@ jobs:
       MODE: "pr"
     secrets: inherit
 
+  run-notebook:
+    name: "Run Notebook"
+    needs: [build-wheel-linux]
+    uses: ./.github/workflows/reusable_run_notebook.yml
+    with:
+      CONCURRENCY: push-linux-x64-${{ github.ref_name }}
+      WHEEL_ARTIFACT_NAME: linux-x64-wheel
+    secrets: inherit
+
   build-examples:
     name: "Build Examples"
     needs: [build-wheel-linux]

diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml
@@ -101,6 +101,7 @@ jobs:
     uses: ./.github/workflows/reusable_checks_rust.yml
     with:
       CONCURRENCY: pr-${{ github.event.pull_request.number }}
+      CHANNEL: pr
     secrets: inherit
 
   python-checks:
@@ -146,6 +147,7 @@ jobs:
       CONCURRENCY: pr-${{ github.event.pull_request.number }}
       PLATFORM: linux-x64
       WHEEL_ARTIFACT_NAME: "linux-x64-wheel-fast"
+      FAST: true
     secrets: inherit
 
   build-js:
@@ -157,7 +159,7 @@ jobs:
     secrets: inherit
 
   build-web:
-    name: "Build and upload web viewer"
+    name: "Build web viewer"
     if: github.event.pull_request.head.repo.owner.login == 'rerun-io'
     uses: ./.github/workflows/reusable_build_web.yml
     with:
@@ -206,20 +208,10 @@ jobs:
       PR_NUMBER: ${{ github.event.pull_request.number }}
     secrets: inherit
 
-  run-notebook:
-    name: "Run Notebook"
-    if: github.event.pull_request.head.repo.owner.login == 'rerun-io'
-    needs: [min-wheel-build]
-    uses: ./.github/workflows/reusable_run_notebook.yml
-    with:
-      CONCURRENCY: pr-${{ github.event.pull_request.number }}
-      WHEEL_ARTIFACT_NAME: linux-x64-wheel-fast
-    secrets: inherit
-
   save-pr-summary:
     name: "Save PR Summary"
     if: github.event.pull_request.head.repo.owner.login == 'rerun-io'
-    needs: [upload-web, run-notebook]
+    needs: [upload-web]
     uses: ./.github/workflows/reusable_pr_summary.yml
     with:
       CONCURRENCY: pr-${{ github.event.pull_request.number }}

diff --git a/.github/workflows/on_push_main.yml b/.github/workflows/on_push_main.yml
@@ -26,6 +26,7 @@ jobs:
     uses: ./.github/workflows/reusable_checks_rust.yml
     with:
       CONCURRENCY: push-${{ github.ref_name }}
+      CHANNEL: main
     secrets: inherit
 
   python_checks:
@@ -77,7 +78,7 @@ jobs:
     secrets: inherit
 
   build-web:
-    name: "Build and upload web viewer"
+    name: "Build web viewer"
     uses: ./.github/workflows/reusable_build_web.yml
     with:
       CONCURRENCY: push-${{ github.ref_name }}
@@ -291,6 +292,8 @@ jobs:
       WHEEL_ARTIFACT_NAME: windows-x64-wheel
     secrets: inherit
 
+  # --------------------------------------------------------------------------
+
   generate-pip-index:
     name: "Generate Pip Index"
     needs:

diff --git a/.github/workflows/reusable_bench.yml b/.github/workflows/reusable_bench.yml
@@ -51,9 +51,6 @@ jobs:
 
     runs-on: ubuntu-latest-16-cores
 
-    container:
-      image: rerunio/ci_docker:0.12.0
-
     steps:
       - uses: actions/checkout@v4
         with:

diff --git a/.github/workflows/reusable_build_web.yml b/.github/workflows/reusable_build_web.yml
@@ -1,4 +1,4 @@
-name: Reusable Build and upload web viewer
+name: Reusable Build web viewer
 
 on:
   workflow_call:
@@ -29,7 +29,7 @@ env:
 
 jobs:
   rs-build-web-viewer:
-    name: Build and upload web viewer
+    name: Build web viewer
     permissions:
       contents: "read"
       id-token: "write"

diff --git a/.github/workflows/reusable_bundle_and_upload_rerun_cpp.yml b/.github/workflows/reusable_bundle_and_upload_rerun_cpp.yml
@@ -20,9 +20,9 @@ jobs:
       id-token: "write"
 
     runs-on: ubuntu-latest
-    # Need container for arrow dependency.
+
     container:
-      image: rerunio/ci_docker:0.12.0
+      image: rerunio/ci_docker:0.12.0 # Need container for arrow dependency.
 
     steps:
       - name: Checkout repository

diff --git a/.github/workflows/reusable_checks.yml b/.github/workflows/reusable_checks.yml
@@ -126,12 +126,6 @@ jobs:
         with:
           python-version: "3.8"
 
-      - name: Install dependencies
-        shell: bash
-        run: |
-          pip install gitignore_parser
-          pip install -r ./scripts/ci/requirements.txt
-
       - name: Rerun lints
         run: pixi run lint-rerun
 
@@ -143,7 +137,7 @@ jobs:
       # - name: Check for zombie TODOs
       #   shell: bash
       #   run: |
-      #     ./scripts/zombie_todos.py --token ${{ secrets.GITHUB_TOKEN }}
+      #     pixi run ./scripts/zombie_todos.py --token ${{ secrets.GITHUB_TOKEN }}
 
       - name: Check for too large files
         shell: bash
@@ -153,12 +147,12 @@ jobs:
       - name: Check Python example requirements
         shell: bash
         run: |
-          ./scripts/ci/check_requirements.py
+          pixi run ./scripts/ci/check_requirements.py
 
       - name: Check Python example thumbnails
         shell: bash
         run: |
-          ./scripts/ci/thumbnails.py check
+          pixi run ./scripts/ci/thumbnails.py check
 
   # ---------------------------------------------------------------------------
 

diff --git a/.github/workflows/reusable_checks_rust.yml b/.github/workflows/reusable_checks_rust.yml
@@ -6,10 +6,9 @@ on:
       CONCURRENCY:
         required: true
         type: string
-      ALL_CHECKS:
+      CHANNEL:
         required: false
-        type: boolean
-        default: false
+        type: string # enum: 'nightly', 'main', or 'pr'
 
 concurrency:
   group: ${{ inputs.CONCURRENCY }}-checks_rust
@@ -68,11 +67,15 @@ jobs:
         run: cargo install --locked --quiet cargo-cranky
 
       - name: Rust checks & tests
-        if: ${{ !inputs.ALL_CHECKS }}
+        if: ${{ inputs.CHANNEL == 'pr' }}
+        run: ./scripts/ci/rust_checks.py --skip-check-individual-crates --skip-test
+
+      - name: Rust checks & tests
+        if: ${{ inputs.CHANNEL == 'main' }}
         run: ./scripts/ci/rust_checks.py --skip-check-individual-crates
 
       - name: Rust all checks & tests
-        if: inputs.ALL_CHECKS
+        if: ${{ inputs.CHANNEL == 'nightly' }}
         run: ./scripts/ci/rust_checks.py
 
   # ---------------------------------------------------------------------------

diff --git a/.github/workflows/reusable_deploy_docs.yml b/.github/workflows/reusable_deploy_docs.yml
@@ -115,8 +115,6 @@ jobs:
     name: Rust
     needs: [py-deploy-docs]
     runs-on: ubuntu-latest-16-cores
-    container:
-      image: rerunio/ci_docker:0.12.0
     steps:
       - name: Show context
         shell: bash

diff --git a/.github/workflows/reusable_run_notebook.yml b/.github/workflows/reusable_run_notebook.yml
@@ -26,7 +26,7 @@ jobs:
     runs-on: ubuntu-latest
 
     container:
-      image: rerunio/ci_docker:0.12.0
+      image: rerunio/ci_docker:0.12.0 # Required to run the wheel or we get "No matching distribution found for attrs>=23.1.0" during `pip install rerun-sdk`
 
     steps:
       - uses: actions/checkout@v4

diff --git a/.github/workflows/reusable_test_wheels.yml b/.github/workflows/reusable_test_wheels.yml
@@ -17,6 +17,10 @@ on:
         required: false
         type: string
         default: ""
+      FAST:
+        required: false
+        type: boolean
+        default: false
 
 concurrency:
   group: ${{ inputs.CONCURRENCY }}-build-wheels
@@ -197,7 +201,7 @@ jobs:
         run: RUST_LOG=debug scripts/run_python_e2e_test.py --no-build # rerun-sdk is already built and installed
 
       - name: Run tests/roundtrips.py
-        if: ${{ inputs.PLATFORM != 'windows-x64' }}
+        if: ${{ inputs.PLATFORM != 'windows-x64' && !inputs.FAST }}
         shell: bash
         # --release so we can inherit from some of the artifacts that maturin has just built before
         # --target x86_64-unknown-linux-gnu because otherwise cargo loses the target cache… even though this is the target anyhow…
@@ -206,7 +210,7 @@ jobs:
           RUST_LOG=debug tests/roundtrips.py --release --target x86_64-unknown-linux-gnu --no-py-build
 
       - name: Run docs/snippets/compare_snippet_output.py
-        if: ${{ inputs.PLATFORM != 'windows-x64' }}
+        if: ${{ inputs.PLATFORM != 'windows-x64' && !inputs.FAST }}
         shell: bash
         # --release so we can inherit from some of the artifacts that maturin has just built before
         # --target x86_64-unknown-linux-gnu because otherwise cargo loses the target cache… even though this is the target anyhow…