Nightly builds (rerun-io#4505)

### What - [x] Rename existing `nightly` to `main` - [x] Add nightly build which runs once per day - [x] Build all examples in nightly build - [x] Build only quick examples in `main` build ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested the web demo (if applicable): * Full build: [app.rerun.io](https://app.rerun.io/pr/4505/index.html) * Partial build: [app.rerun.io](https://app.rerun.io/pr/4505/index.html?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) - Useful for quick testing when changes do not affect examples in any way * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG - [PR Build Summary](https://build.rerun.io/pr/4505) - [Docs preview](https://rerun.io/preview/0d1b795a25d0dc88d35f215e37bf84e6caf2b3cf/docs)  - [Examples preview](https://rerun.io/preview/0d1b795a25d0dc88d35f215e37bf84e6caf2b3cf/examples)  - [Recent benchmark results](https://build.rerun.io/graphs/crates.html) - [Wasm size tracking](https://build.rerun.io/graphs/sizes.html) --------- Co-authored-by: Emil Ernerfeldt <[email protected]>
m-decoster · Dec 14, 2023 · 2e99a7b · 2e99a7b
1 parent 3221f6c
commit 2e99a7b
Show file tree

Hide file tree

Showing 46 changed files with 703 additions and 597 deletions.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -16,8 +16,9 @@ To get an auto-generated PR description you can put "copilot:summary" or "copilo
 * [ ] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
 * [ ] I've included a screenshot or gif (if applicable)
 * [ ] I have tested the web demo (if applicable):
-  * Full build: [app.rerun.io](https://app.rerun.io/pr/{{ pr.number }}/index.html)
-  * Partial build: [app.rerun.io](https://app.rerun.io/pr/{{ pr.number }}/index.html?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) - Useful for quick testing when changes do not affect examples in any way
+  * Using newly built examples: [app.rerun.io](https://app.rerun.io/pr/{{pr.number}}/index.html)
+  * Using examples from latest `main` build: [app.rerun.io](https://app.rerun.io/pr/{{pr.number}}/index.html?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
+  * Using full set of examples from `nightly` build: [app.rerun.io](https://app.rerun.io/pr/{{pr.number}}/index.html?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
 * [ ] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG
 
 - [PR Build Summary](https://build.rerun.io/pr/{{ pr.number }})

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -0,0 +1,55 @@
+name: Nightly
+
+on:
+  workflow_dispatch:
+  schedule:
+    # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule
+    # 12:15 UTC, every day
+    - cron: "15 12 * * *"
+
+jobs:
+  build-web:
+    name: "Build Web"
+    uses: ./.github/workflows/reusable_build_web.yml
+    with:
+      CONCURRENCY: nightly
+      CHANNEL: nightly
+    secrets: inherit
+
+  upload-web:
+    name: "Upload Web"
+    needs: [build-web]
+    uses: ./.github/workflows/reusable_upload_web.yml
+    with:
+      CONCURRENCY: nightly
+      NIGHTLY: true
+    secrets: inherit
+
+  build-wheel-linux:
+    name: "Build & Upload Wheels"
+    uses: ./.github/workflows/reusable_build_and_upload_wheels.yml
+    with:
+      CONCURRENCY: nightly-linux
+      PLATFORM: linux
+      WHEEL_ARTIFACT_NAME: linux-wheel
+      MODE: "pr"
+    secrets: inherit
+
+  build-examples:
+    name: "Build Examples"
+    needs: [build-wheel-linux]
+    uses: ./.github/workflows/reusable_build_examples.yml
+    with:
+      CONCURRENCY: nightly
+      CHANNEL: nightly
+      WHEEL_ARTIFACT_NAME: linux-wheel
+    secrets: inherit
+
+  upload-examples:
+    name: "Upload Examples"
+    needs: [build-examples]
+    uses: ./.github/workflows/reusable_upload_examples.yml
+    with:
+      CONCURRENCY: nightly
+      NIGHTLY: true
+    secrets: inherit
diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml
@@ -111,6 +111,7 @@ jobs:
     uses: ./.github/workflows/reusable_build_web.yml
     with:
       CONCURRENCY: pr-${{ github.event.pull_request.number }}
+      CHANNEL: main
     secrets: inherit
 
   upload-web:
@@ -130,13 +131,14 @@ jobs:
     uses: ./.github/workflows/reusable_build_examples.yml
     with:
       CONCURRENCY: pr-${{ github.event.pull_request.number }}
+      CHANNEL: main
       WHEEL_ARTIFACT_NAME: linux-wheel-fast
     secrets: inherit
 
   track-sizes:
     name: "Track Sizes"
     if: github.event.pull_request.head.repo.owner.login == 'rerun-io'
-    needs: [build-examples]
+    needs: [build-web, build-examples]
     uses: ./.github/workflows/reusable_track_size.yml
     with:
       CONCURRENCY: push-${{ github.ref_name }}

diff --git a/.github/workflows/on_push_main.yml b/.github/workflows/on_push_main.yml
@@ -56,7 +56,7 @@ jobs:
     uses: ./.github/workflows/reusable_deploy_docs.yml
     with:
       CONCURRENCY: push-${{ github.ref_name }}
-      PY_AND_CPP_DOCS_VERSION_NAME: "nightly"
+      PY_AND_CPP_DOCS_VERSION_NAME: "main"
       UPDATE_LATEST: false
     secrets: inherit
 
@@ -65,6 +65,7 @@ jobs:
     uses: ./.github/workflows/reusable_build_web.yml
     with:
       CONCURRENCY: push-${{ github.ref_name }}
+      CHANNEL: main
     secrets: inherit
 
   upload-web:
@@ -81,12 +82,13 @@ jobs:
     uses: ./.github/workflows/reusable_build_examples.yml
     with:
       CONCURRENCY: push-${{ github.ref_name }}
+      CHANNEL: main
       WHEEL_ARTIFACT_NAME: linux-wheel
     secrets: inherit
 
   track-sizes:
     name: "Track Sizes"
-    needs: [build-examples]
+    needs: [build-web, build-examples]
     uses: ./.github/workflows/reusable_track_size.yml
     with:
       CONCURRENCY: push-${{ github.ref_name }}

diff --git a/.github/workflows/reusable_build_examples.yml b/.github/workflows/reusable_build_examples.yml
@@ -9,13 +9,15 @@ on:
       WHEEL_ARTIFACT_NAME:
         required: true
         type: string
+      CHANNEL: # `nightly` or `main`
+        required: true
+        type: string
 
 concurrency:
   group: ${{ inputs.CONCURRENCY }}-build-examples
   cancel-in-progress: true
 
 env:
-  PYTHON_VERSION: "3.8"
   # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses
   # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html
   # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html
@@ -33,9 +35,6 @@ jobs:
 
     runs-on: ubuntu-latest-16-cores
 
-    container:
-      image: rerunio/ci_docker:0.11.0
-
     steps:
       - uses: actions/checkout@v4
         with:
@@ -49,20 +48,26 @@ jobs:
           workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
           service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
 
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.6.0
+
       - name: Download Wheel
         uses: actions/download-artifact@v3
         with:
           name: ${{ inputs.WHEEL_ARTIFACT_NAME }}
           path: wheel
 
-      - uses: prefix-dev/[email protected]
-        with:
-          pixi-version: v0.6.0
+      - name: Install example dependencies
+        shell: bash
+        run: |
+          pixi run pip install \
+            -r scripts/ci/requirements-examples-${{ inputs.CHANNEL }}.txt \
+            --no-input
 
       - name: Install Python dependencies and wheel
         shell: bash
         run: |
-          pixi run pip install -r scripts/ci/requirements-web-demo.txt
           pixi run pip uninstall rerun-sdk -y
           pixi run pip install deprecated numpy>=1.23 pyarrow==10.0.1 pytest==7.1.2
           pixi run pip install rerun-sdk --no-index --find-links wheel
@@ -84,7 +89,9 @@ jobs:
       - name: Build examples
         shell: bash
         run: |
-          pixi run build-examples example_data
+          pixi run build-examples rrd \
+            --channel ${{ inputs.CHANNEL }} \
+            example_data
 
       - name: Upload assets
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/reusable_build_web.yml b/.github/workflows/reusable_build_web.yml
@@ -10,10 +10,9 @@ on:
         required: false
         type: string
         default: "prerelease"
-      EXTRA_FLAGS:
-        required: false
+      CHANNEL: # `nightly` or `main`
+        required: true
         type: string
-        default: ""
 
 concurrency:
   group: ${{ inputs.CONCURRENCY }}-build-web
@@ -54,21 +53,29 @@ jobs:
           workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
           service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
 
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.6.0
+
       - name: Build web-viewer (release)
         shell: bash
         run: |
-          cargo run --locked -p re_build_web_viewer -- --release ${{ inputs.EXTRA_FLAGS }}
+          if [ ${{ inputs.CHANNEL }} = "nightly" ]; then
+            export DEFAULT_EXAMPLES_MANIFEST_URL="https://app.rerun.io/version/nightly/examples_manifest.json"
+          fi
+          cargo run --locked -p re_build_web_viewer -- --release
 
       # We build a single manifest pointing to the `commit`
-      # All the `pr`, `nightly`, release tag, etc. variants will always just point to the resolved commit
+      # All the `pr`, `main`, release tag, etc. variants will always just point to the resolved commit
       - name: Build examples manifest
         shell: bash
         run: |
           full_commit="${{ (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha }}"
           sha="$(echo $full_commit | cut -c1-7)"
 
-          cargo run --locked -p re_build_examples_manifest -- \
+          pixi run build-examples manifest \
             --base-url "https://app.rerun.io/commit/$sha" \
+            --channel "${{ inputs.CHANNEL }}" \
             web_viewer/examples_manifest.json
 
       - name: Upload web assets

diff --git a/.github/workflows/reusable_publish_web.yml b/.github/workflows/reusable_publish_web.yml
@@ -107,16 +107,19 @@ jobs:
       - name: Build examples
         shell: bash
         run: |
-          pixi run build-examples web_viewer/examples
+          pixi run build-examples rrd \
+            --channel "main" \
+            web_viewer/examples
 
       - name: Build examples manifest
         shell: bash
         run: |
           full_commit="${{ (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha }}"
           sha="$(echo $full_commit | cut -c1-7)"
 
-          pixi run build-examples-manifest \
+          pixi run build-examples manifest \
             --base-url "https://app.rerun.io/commit/$sha" \
+            --channel "main" \
             web_viewer/examples_manifest.json
 
       - name: Upload app.rerun.io (versioned)

diff --git a/.github/workflows/reusable_upload_examples.yml b/.github/workflows/reusable_upload_examples.yml
@@ -21,6 +21,10 @@ on:
       PR_NUMBER:
         type: string
         default: ""
+      NIGHTLY:
+        required: false
+        type: boolean
+        default: false
 
 concurrency:
   group: ${{ inputs.CONCURRENCY }}-upload-examples
@@ -61,6 +65,7 @@ jobs:
           echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT"
 
       - name: "Upload examples (commit)"
+        if: ${{ !inputs.NIGHTLY }}
         uses: google-github-actions/upload-cloud-storage@v1
         with:
           path: "example_data"
@@ -93,13 +98,12 @@ jobs:
           headers: |-
             cache-control: no-cache, max-age=0
 
-      - name: "Upload examples (nightly)"
-        # TEMP: Tracking `main` until we can do actual nightly builds.
+      - name: "Upload examples (main)"
         if: github.ref == 'refs/heads/main'
         uses: google-github-actions/upload-cloud-storage@v1
         with:
           path: "example_data"
-          destination: "rerun-web-viewer/version/nightly/examples"
+          destination: "rerun-web-viewer/version/main/examples"
           parent: false
           headers: |-
             cache-control: no-cache, max-age=0
@@ -113,3 +117,13 @@ jobs:
           parent: false
           headers: |-
             cache-control: no-cache, max-age=0
+
+      - name: "Upload examples (nightly)"
+        if: ${{ inputs.NIGHTLY }}
+        uses: google-github-actions/upload-cloud-storage@v1
+        with:
+          path: "example_data"
+          destination: "rerun-web-viewer/version/nightly/examples"
+          parent: false
+          headers: |-
+            cache-control: no-cache, max-age=0
diff --git a/.github/workflows/reusable_upload_web.yml b/.github/workflows/reusable_upload_web.yml
@@ -22,6 +22,10 @@ on:
         required: false
         type: string
         default: ""
+      NIGHTLY:
+        required: false
+        type: boolean
+        default: false
 
 concurrency:
   group: ${{ inputs.CONCURRENCY }}-upload-web
@@ -62,6 +66,7 @@ jobs:
           echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT"
 
       - name: "Upload web-viewer (commit)"
+        if: ${{ !inputs.NIGHTLY }}
         uses: google-github-actions/upload-cloud-storage@v1
         with:
           path: "web_viewer"
@@ -94,13 +99,12 @@ jobs:
           headers: |-
             cache-control: no-cache, max-age=0
 
-      - name: "Upload web-viewer (nightly)"
-        # TEMP: Tracking `main` until we can do actual nightly builds.
+      - name: "Upload web-viewer (main)"
         if: github.ref == 'refs/heads/main'
         uses: google-github-actions/upload-cloud-storage@v1
         with:
           path: "web_viewer"
-          destination: "rerun-web-viewer/version/nightly"
+          destination: "rerun-web-viewer/version/main"
           parent: false
           headers: |-
             cache-control: no-cache, max-age=0
@@ -114,3 +118,13 @@ jobs:
           parent: false
           headers: |-
             cache-control: no-cache, max-age=0
+
+      - name: "Upload web-viewer (nightly)"
+        if: ${{ inputs.NIGHTLY }}
+        uses: google-github-actions/upload-cloud-storage@v1
+        with:
+          path: "web_viewer"
+          destination: "rerun-web-viewer/version/nightly"
+          parent: false
+          headers: |-
+            cache-control: no-cache, max-age=0
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
@@ -179,8 +179,7 @@ Update instructions:
 | re_build_info              | Information about the build. Use together with re_build_tools |
 | re_build_tools             | build.rs helpers for generating build info                    |
 | re_types_builder           | Generates code for Rerun's SDKs from flatbuffers definitions. |
-| re_build_examples          | Build rerun example RRD files                                 |
-| re_build_examples_manifest | Build the rerun examples manifest JSON file                   |
+| re_build_examples          | Build rerun example RRD files and manifest                    |
 | re_build_web_viewer        | Build the rerun web-viewer Wasm from source                   |