Move many CI jobs from main to nightly (rerun-io#5937)

### What * A huge part of rerun-io#5332 This moves _a lot_ of jobs from `main` to nightly: * Building and testing wheels, `rerun_c` and `rerun-cli` for all platforms (keeping just linux-x86 on `main`) * Doing a pre-release * Benchmarks * Clean build This should reduce the billable time on `main` significantly. I've mostly just moved the jobs (leaving a few behind) and changing `CONCURRENCY:` to `nightly` everywhere. ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested the web demo (if applicable): * Using examples from latest `main` build: [rerun.io/viewer](https://rerun.io/viewer/pr/5937?manifest_url=https://app.rerun.io/version/main/examples_manifest.json) * Using full set of examples from `nightly` build: [rerun.io/viewer](https://rerun.io/viewer/pr/5937?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG * [x] If applicable, add a new check to the [release checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)! - [PR Build Summary](https://build.rerun.io/pr/5937) - [Recent benchmark results](https://build.rerun.io/graphs/crates.html) - [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)
m-decoster · Apr 12, 2024 · 78d46da · 78d46da
1 parent 41986fa
commit 78d46da
Show file tree

Hide file tree

Showing 3 changed files with 356 additions and 282 deletions.
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -38,6 +38,22 @@ jobs:
       CONCURRENCY: nightly
     secrets: inherit
 
+  # Check that a CLEAN container with just `cargo` on it can build rerun:
+  clean-build:
+    name: cargo build on clean container
+    strategy:
+      matrix:
+        os: [ubuntu-latest-16-cores, macos-latest, windows-latest-8-cores]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: 1.76.0
+
+      - run: cargo build -p rerun
+        shell: bash
+
   build-web:
     name: "Build web viewer"
     uses: ./.github/workflows/reusable_build_web.yml
@@ -55,28 +71,228 @@ jobs:
       NIGHTLY: true
     secrets: inherit
 
-  build-wheel-linux:
-    name: "Build & Upload Wheels"
+  # ---------------------------------------------------------------------------
+  # Build wheels:
+
+  build-wheel-linux-arm64:
+    needs: [checks]
+    name: "Linux-arm64: Build & Upload Wheels"
+    uses: ./.github/workflows/reusable_build_and_upload_wheels.yml
+    with:
+      CONCURRENCY: nightly
+      PLATFORM: linux-arm64
+      WHEEL_ARTIFACT_NAME: linux-arm64-wheel
+      MODE: "pypi"
+    secrets: inherit
+
+  build-wheel-linux-x64:
+    needs: [checks]
+    name: "Linux-x64: Build & Upload Wheels"
+    uses: ./.github/workflows/reusable_build_and_upload_wheels.yml
+    with:
+      CONCURRENCY: nightly
+      PLATFORM: linux-x64
+      WHEEL_ARTIFACT_NAME: linux-x64-wheel
+      MODE: "pypi"
+    secrets: inherit
+
+  build-wheel-macos-arm64:
+    needs: [checks]
+    name: "Macos-arm64: Build & Upload Wheels"
     uses: ./.github/workflows/reusable_build_and_upload_wheels.yml
     with:
-      CONCURRENCY: nightly-linux
+      CONCURRENCY: nightly
+      PLATFORM: macos-arm64
+      WHEEL_ARTIFACT_NAME: macos-arm64-wheel
+      MODE: "pypi"
+    secrets: inherit
+
+  build-wheel-macos-x64:
+    needs: [checks]
+    name: "Macos-x64: Build & Upload Wheels"
+    uses: ./.github/workflows/reusable_build_and_upload_wheels.yml
+    with:
+      CONCURRENCY: nightly
+      PLATFORM: macos-x64
+      WHEEL_ARTIFACT_NAME: "macos-x64-wheel"
+      MODE: "pypi"
+    secrets: inherit
+
+  build-wheel-windows-x64:
+    needs: [checks]
+    name: "Windows-x64: Build & Upload Wheels"
+    uses: ./.github/workflows/reusable_build_and_upload_wheels.yml
+    with:
+      CONCURRENCY: nightly
+      PLATFORM: windows-x64
+      WHEEL_ARTIFACT_NAME: windows-x64-wheel
+      MODE: "pypi"
+    secrets: inherit
+
+  # ---------------------------------------------------------------------------
+  # Test wheels:
+
+  test-wheel-linux-arm64:
+    needs: [checks, build-wheel-linux-arm64]
+    name: "linux-arm64: Test Wheels"
+    uses: ./.github/workflows/reusable_test_wheels.yml
+    with:
+      CONCURRENCY: nightly
+      PLATFORM: linux-arm64
+      WHEEL_ARTIFACT_NAME: linux-arm64-wheel
+    secrets: inherit
+
+  test-wheel-linux-x64:
+    needs: [checks, build-wheel-linux-x64]
+    name: "Linux-x64: Test Wheels"
+    uses: ./.github/workflows/reusable_test_wheels.yml
+    with:
+      CONCURRENCY: nightly
       PLATFORM: linux-x64
       WHEEL_ARTIFACT_NAME: linux-x64-wheel
-      MODE: "pr"
     secrets: inherit
 
+  test-wheel-macos-arm64:
+    needs: [checks, build-wheel-macos-arm64]
+    name: "macos-arm64: Test Wheels"
+    uses: ./.github/workflows/reusable_test_wheels.yml
+    with:
+      CONCURRENCY: nightly
+      PLATFORM: macos-arm64
+      WHEEL_ARTIFACT_NAME: macos-arm64-wheel
+    secrets: inherit
+
+  test-wheel-macos-x64:
+    needs: [checks, build-wheel-macos-x64]
+    name: "macos-x64: Test Wheels"
+    uses: ./.github/workflows/reusable_test_wheels.yml
+    with:
+      CONCURRENCY: nightly
+      PLATFORM: macos-x64
+      WHEEL_ARTIFACT_NAME: macos-x64-wheel
+    secrets: inherit
+
+  test-wheel-windows-x64:
+    needs: [checks, build-wheel-windows-x64]
+    name: "Windows-x64: Test Wheels"
+    uses: ./.github/workflows/reusable_test_wheels.yml
+    with:
+      CONCURRENCY: nightly
+      PLATFORM: windows-x64
+      WHEEL_ARTIFACT_NAME: windows-x64-wheel
+    secrets: inherit
+
+  # -----------------------------------------------------------------------------------
+  # Build rerun_c library binaries:
+
+  build-rerun_c-and-upload-linux-arm64:
+    needs: [checks]
+    name: "Linux-Arm64: Build & Upload rerun_c"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml
+    with:
+      CONCURRENCY: nightly
+      PLATFORM: linux-arm64
+    secrets: inherit
+
+  build-rerun_c-and-upload-linux-x64:
+    needs: [checks]
+    name: "Linux-x64: Build & Upload rerun_c"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml
+    with:
+      CONCURRENCY: nightly
+      PLATFORM: linux-x64
+    secrets: inherit
+
+  build-rerun_c-and-upload-macos-x64:
+    needs: [checks]
+    name: "Mac-Intel: Build & Upload rerun_c"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml
+    with:
+      CONCURRENCY: nightly
+      PLATFORM: macos-x64
+    secrets: inherit
+
+  build-rerun_c-and-upload-macos-arm64:
+    needs: [checks]
+    name: "Mac-Arm64: Build & Upload rerun_c"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml
+    with:
+      CONCURRENCY: nightly
+      PLATFORM: macos-arm64
+    secrets: inherit
+
+  build-rerun_c-and-upload-windows-x64:
+    needs: [checks]
+    name: "Windows-x64: Build & Upload rerun_c"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml
+    with:
+      CONCURRENCY: nightly
+      PLATFORM: windows-x64
+    secrets: inherit
+
+  # -----------------------------------------------------------------------------------
+  # TODO(emilk): build and test one additional platorm, picked at random
+  # Build rerun-cli (rerun binaries):
+
+  build-rerun-cli-and-upload-linux-arm64:
+    needs: [checks]
+    name: "Linux-arm64: Build & Upload rerun-cli"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
+    with:
+      CONCURRENCY: nightly
+      PLATFORM: linux-arm64
+    secrets: inherit
+
+  build-rerun-cli-and-upload-linux-x64:
+    needs: [checks]
+    name: "Linux-x64: Build & Upload rerun-cli"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
+    with:
+      CONCURRENCY: nightly
+      PLATFORM: linux-x64
+    secrets: inherit
+
+  build-rerun-cli-and-upload-macos-x64:
+    needs: [checks]
+    name: "Mac-x64: Build & Upload rerun-cli"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
+    with:
+      CONCURRENCY: nightly
+      PLATFORM: macos-x64
+    secrets: inherit
+
+  build-rerun-cli-and-upload-macos-arm64:
+    needs: [checks]
+    name: "Mac-arm64: Build & Upload rerun-cli"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
+    with:
+      CONCURRENCY: nightly
+      PLATFORM: macos-arm64
+    secrets: inherit
+
+  build-rerun-cli-and-upload-windows-x64:
+    needs: [checks]
+    name: "Windows-x64: Build & Upload rerun-cli"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
+    with:
+      CONCURRENCY: nightly
+      PLATFORM: windows-x64
+    secrets: inherit
+
+  # ---------------------------------------------------------------------------
+
   run-notebook:
     name: "Run Notebook"
-    needs: [build-wheel-linux]
+    needs: [build-wheel-linux-x64]
     uses: ./.github/workflows/reusable_run_notebook.yml
     with:
-      CONCURRENCY: push-linux-x64-${{ github.ref_name }}
+      CONCURRENCY: nightly
       WHEEL_ARTIFACT_NAME: linux-x64-wheel
     secrets: inherit
 
   build-examples:
     name: "Build Examples"
-    needs: [build-wheel-linux]
+    needs: [build-wheel-linux-x64]
     uses: ./.github/workflows/reusable_build_examples.yml
     with:
       CONCURRENCY: nightly
@@ -92,3 +308,131 @@ jobs:
       CONCURRENCY: nightly
       NIGHTLY: true
     secrets: inherit
+
+  benches:
+    name: Benchmarks
+    uses: ./.github/workflows/reusable_bench.yml
+    with:
+      CONCURRENCY: nightly
+      SAVE_BENCHES: true
+      BENCH_NAME: main # We currently only run benches nightly, but we used to run them on main
+      COMPARE_TO: main # We currently only run benches nightly, but we used to run them on main
+    secrets: inherit
+
+  # --------------------------------------------------------------------------
+  # Release:
+
+  generate-pip-index:
+    name: "Generate Pip Index"
+    needs:
+      [
+        build-wheel-linux-arm64,
+        build-wheel-linux-x64,
+        build-wheel-macos-arm64,
+        build-wheel-macos-x64,
+        build-wheel-windows-x64,
+      ]
+    uses: ./.github/workflows/reusable_pip_index.yml
+    with:
+      CONCURRENCY: nightly
+    secrets: inherit
+
+  bundle-and-upload-rerun_cpp:
+    name: "Bundle and upload rerun_cpp_sdk.zip"
+    needs:
+      [
+        build-rerun_c-and-upload-linux-arm64,
+        build-rerun_c-and-upload-linux-x64,
+        build-rerun_c-and-upload-macos-arm64,
+        build-rerun_c-and-upload-macos-x64,
+        build-rerun_c-and-upload-windows-x64,
+      ]
+    uses: ./.github/workflows/reusable_bundle_and_upload_rerun_cpp.yml
+    secrets: inherit
+
+  pre-release:
+    name: Pre Release
+    concurrency: NIGHTLY
+    needs:
+      [
+        build-rerun-cli-and-upload-linux-arm64,
+        build-rerun-cli-and-upload-linux-x64,
+        build-rerun-cli-and-upload-macos-arm64,
+        build-rerun-cli-and-upload-macos-x64,
+        build-rerun-cli-and-upload-windows-x64,
+        build-rerun_c-and-upload-linux-arm64,
+        build-rerun_c-and-upload-linux-x64,
+        build-rerun_c-and-upload-macos-arm64,
+        build-rerun_c-and-upload-macos-x64,
+        build-rerun_c-and-upload-windows-x64,
+        bundle-and-upload-rerun_cpp,
+        generate-pip-index,
+        upload-web,
+      ]
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: Add SHORT_SHA env property with commit short sha
+        shell: bash
+        run: echo "SHORT_SHA=`echo ${{github.sha}} | cut -c1-7`" >> $GITHUB_ENV
+
+      # First delete the old prerelease. If we don't do this, we don't get things like
+      # proper source-archives and changelog info.
+      # https://github.com/dev-drprasad/delete-tag-and-release
+      - uses: dev-drprasad/[email protected]
+        with:
+          tag_name: prerelease
+          delete_release: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      # Create the actual prerelease
+      # https://github.com/ncipollo/release-action
+      - name: GitHub Release
+        uses: ncipollo/[email protected]
+        with:
+          body: |
+            This is a prerelease. It is not intended for production use.
+            Please report any issues you find.
+
+            ## Example Hosted App
+            https://rerun.io/viewer/commit/${{ env.SHORT_SHA }}
+
+            ## Wheels can be installed with:
+            ```
+            pip install --pre -f https://build.rerun.io/commit/${{ env.SHORT_SHA }}/wheels --upgrade rerun-sdk
+            ```
+            or
+            ```
+            pip install --pre -f https://github.com/rerun-io/rerun/releases/download/prerelease --upgrade rerun-sdk
+            ```
+
+            ## CMake fetch-content for C++ SDK
+            ```
+            include(FetchContent)
+            FetchContent_Declare(rerun_sdk URL https://build.rerun.io/commit/${{ env.SHORT_SHA }}/rerun_cpp_sdk.zip)
+            FetchContent_MakeAvailable(rerun_sdk)
+            ```
+            or
+            ```
+            include(FetchContent)
+            FetchContent_Declare(rerun_sdk URL https://github.com/rerun-io/rerun/releases/download/prerelease/rerun_cpp_sdk.zip)
+            FetchContent_MakeAvailable(rerun_sdk)
+            ```
+
+          prerelease: true
+          name: "Development Build"
+          tag: "prerelease"
+          token: ${{ secrets.GITHUB_TOKEN }}
+          generateReleaseNotes: false
+          allowUpdates: true
+          removeArtifacts: true
+          replacesArtifacts: true
+
+  sync-release-assets:
+    needs: [pre-release]
+    name: "Sync pre-release assets & build.rerun.io"
+    uses: ./.github/workflows/reusable_sync_release_assets.yml
+    with:
+      CONCURRENCY: nightly
+      RELEASE_VERSION: prerelease
+    secrets: inherit