From 85cb1025b98981a71e2fdc6d0b8f450e3da29618 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 16 Apr 2024 18:28:57 +0200 Subject: [PATCH] Fix nightly ci (#5973) ### What * Fixes #5945 * various wheel build issues * notebook issues * Fixes #3924 ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested the web demo (if applicable): * Using examples from latest `main` build: [rerun.io/viewer](https://rerun.io/viewer/pr/5973?manifest_url=https://app.rerun.io/version/main/examples_manifest.json) * Using full set of examples from `nightly` build: [rerun.io/viewer](https://rerun.io/viewer/pr/5973?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG * [x] If applicable, add a new check to the [release checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)! - [PR Build Summary](https://build.rerun.io/pr/5973) - [Recent benchmark results](https://build.rerun.io/graphs/crates.html) - [Wasm size tracking](https://build.rerun.io/graphs/sizes.html) To run all checks from `main`, comment on the PR with `@rerun-bot full-check`. --- .github/workflows/reusable_checks_rust.yml | 26 ++++++++++++++++++ .github/workflows/reusable_run_notebook.yml | 30 ++++++++++----------- .github/workflows/reusable_test_wheels.yml | 25 +++++++---------- docs/snippets/snippets.toml | 3 +++ scripts/run_python_e2e_test.py | 9 ++----- tests/roundtrips.py | 3 ++- 6 files changed, 58 insertions(+), 38 deletions(-) diff --git a/.github/workflows/reusable_checks_rust.yml b/.github/workflows/reusable_checks_rust.yml index 575cb403e997..7044158007d2 100644 --- a/.github/workflows/reusable_checks_rust.yml +++ b/.github/workflows/reusable_checks_rust.yml @@ -78,3 +78,29 @@ jobs: - name: Rust all checks & tests if: ${{ inputs.CHANNEL == 'nightly' }} run: pixi run rs-check + + # Run some basics tests on Mac and Windows + mac-windows-tests: + name: Test on macOS and Windows + if: ${{ inputs.CHANNEL == 'nightly' }} + strategy: + matrix: + include: + - os: macos-latest + name: macos + - os: windows-latest-8-cores + name: windows + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + - name: Set up Rust + uses: ./.github/actions/setup-rust + with: + cache_key: "build-${{ matrix.name }}" + save_cache: true + workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} + + - name: cargo test --all-targets --all-features + run: cargo test --all-targets --all-features diff --git a/.github/workflows/reusable_run_notebook.yml b/.github/workflows/reusable_run_notebook.yml index e54512d6e3c7..795a3a22267a 100644 --- a/.github/workflows/reusable_run_notebook.yml +++ b/.github/workflows/reusable_run_notebook.yml @@ -33,37 +33,37 @@ jobs: with: ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - name: Download Wheel Artifact + - uses: prefix-dev/setup-pixi@v0.5.2 + with: + pixi-version: v0.19.0 + environments: wheel-test + + - name: Download Wheel uses: actions/download-artifact@v4 with: name: ${{ inputs.WHEEL_ARTIFACT_NAME }} - path: wheel_artifact + path: wheel - - name: Install wheel dependencies - # First we install the dependencies manually so we can use `--no-index` when installing the wheel. - # This needs to be a separate step for some reason or the following step fails - # TODO(jleibs): pull these deps from pyproject.toml + - name: Get version + id: get-version shell: bash run: | - pip install deprecated 'numpy>=1.23,<2' pillow>=9.5.0 pyarrow>=14.0.2 pytest==7.1.2 + pixi run -e wheel-test 'echo "wheel_version=$(python scripts/ci/crates.py get-version)"' >> "$GITHUB_OUTPUT" - - name: Install downloaded wheel_artifact + - name: Install built wheel # Now install the wheel using a specific version and --no-index to guarantee we get the version from # the pre-dist folder. Note we don't use --force-reinstall here because --no-index means it wouldn't # find the dependencies to reinstall them. shell: bash run: | - pip install rerun-sdk --no-index --find-links wheel_artifact + pixi run -e wheel-test pip uninstall rerun-sdk + pixi run -e wheel-test pip install rerun-sdk==${{ steps.get-version.outputs.wheel_version }} --no-index --find-links wheel - name: Install Deps - shell: bash - run: | - pip install -r examples/python/notebook/requirements.txt + run: pixi run -e wheel-test pip install -r examples/python/notebook/requirements.txt - name: Create notebook - shell: bash - run: | - jupyter nbconvert --to=html --ExecutePreprocessor.enabled=True examples/python/notebook/cube.ipynb --output /tmp/cube.html + run: pixi run -e wheel-test jupyter nbconvert --to=html --ExecutePreprocessor.enabled=True examples/python/notebook/cube.ipynb --output /tmp/cube.html - id: "auth" uses: google-github-actions/auth@v2 diff --git a/.github/workflows/reusable_test_wheels.yml b/.github/workflows/reusable_test_wheels.yml index fa8702f0d976..5d5b15bdeed4 100644 --- a/.github/workflows/reusable_test_wheels.yml +++ b/.github/workflows/reusable_test_wheels.yml @@ -87,12 +87,12 @@ jobs: container="null" ;; macos-arm64) - runner="macos-latest-large" # See https://github.blog/2023-10-02-introducing-the-new-apple-silicon-powered-m1-macos-larger-runner-for-github-actions/ + runner="macos-13-xlarge" # This is an Arm vm, https://docs.github.com/en/actions/using-github-hosted-runners/about-larger-runners/about-larger-runners#about-macos-larger-runners target="aarch64-apple-darwin" container="null" ;; macos-x64) - runner="macos-latest-large" # See https://github.blog/2023-10-02-introducing-the-new-apple-silicon-powered-m1-macos-larger-runner-for-github-actions/ + runner="macos-13-large" # This is an Intel vm, see https://docs.github.com/en/actions/using-github-hosted-runners/about-larger-runners/about-larger-runners#about-macos-larger-runners target="x86_64-apple-darwin" container="null" ;; @@ -158,7 +158,7 @@ jobs: id: get-version shell: bash run: | - echo "wheel_version=$(pixi run scripts/ci/crates.py get-version)" >> "$GITHUB_OUTPUT" + pixi run -e wheel-test 'echo "wheel_version=$(python scripts/ci/crates.py get-version)"' >> "$GITHUB_OUTPUT" - name: Install built wheel # Now install the wheel using a specific version and --no-index to guarantee we get the version from @@ -181,23 +181,18 @@ jobs: run: cd rerun_py/tests && pixi run -e wheel-test pytest -c ../pyproject.toml - name: Run e2e test - shell: bash - run: pixi run -e wheel-test RUST_LOG=debug scripts/run_python_e2e_test.py --no-build # rerun-sdk is already built and installed + run: pixi run -e wheel-test RUST_LOG=debug python scripts/run_python_e2e_test.py --no-build # rerun-sdk is already built and installed - name: Run tests/roundtrips.py - if: ${{ inputs.PLATFORM != 'windows-x64' && !inputs.FAST }} - shell: bash + if: ${{ !inputs.FAST }} # --release so we can inherit from some of the artifacts that maturin has just built before - # --target x86_64-unknown-linux-gnu because otherwise cargo loses the target cache… even though this is the target anyhow… + # explicit target because otherwise cargo loses the target cache… even though this is the target anyhow… # --no-py-build because rerun-sdk is already built and installed - run: | - pixi run -e wheel-test RUST_LOG=debug tests/roundtrips.py --release --target x86_64-unknown-linux-gnu --no-py-build + run: pixi run -e wheel-test RUST_LOG=debug python tests/roundtrips.py --release --target ${{ needs.set-config.outputs.TARGET }} --no-py-build - name: Run docs/snippets/compare_snippet_output.py - if: ${{ inputs.PLATFORM != 'windows-x64' && !inputs.FAST }} - shell: bash + if: ${{ !inputs.FAST }} # --release so we can inherit from some of the artifacts that maturin has just built before - # --target x86_64-unknown-linux-gnu because otherwise cargo loses the target cache… even though this is the target anyhow… + # explicit target because otherwise cargo loses the target cache… even though this is the target anyhow… # --no-py-build because rerun-sdk is already built and installed - run: | - pixi run -e wheel-test RUST_LOG=debug docs/snippets/compare_snippet_output.py --release --target x86_64-unknown-linux-gnu --no-py-build + run: pixi run -e wheel-test RUST_LOG=debug python docs/snippets/compare_snippet_output.py --release --target ${{ needs.set-config.outputs.TARGET }} --no-py-build diff --git a/docs/snippets/snippets.toml b/docs/snippets/snippets.toml index 9ccb452f5d2e..c825a57ce6a8 100644 --- a/docs/snippets/snippets.toml +++ b/docs/snippets/snippets.toml @@ -38,6 +38,9 @@ asset3d_out_of_tree = [ # float issues since calculation is done slightly differ "py", "rust", ] +bar_chart = [ # On Windows this logs f64 instead of u64 unless a numpy array with explicit type is used. + "py", +] mesh3d_partial_updates = ["cpp", "py", "rust"] # float precision issues pinhole_simple = [ # TODO(#3206): examples use different RNGs "cpp", diff --git a/scripts/run_python_e2e_test.py b/scripts/run_python_e2e_test.py index 6080e1a1a5e8..e3bd74d0ca54 100755 --- a/scripts/run_python_e2e_test.py +++ b/scripts/run_python_e2e_test.py @@ -81,20 +81,15 @@ def main() -> None: def run_example(example: str, extra_args: list[str]) -> None: - # sys.executable: the absolute path of the executable binary for the Python interpreter - python_executable = sys.executable - if python_executable is None: - python_executable = "python3" - env = os.environ.copy() env["RERUN_STRICT"] = "1" env["RERUN_PANIC_ON_WARN"] = "1" - cmd = [python_executable, "-m", "rerun", "--port", str(PORT), "--test-receive"] + cmd = ["python", "-m", "rerun", "--port", str(PORT), "--test-receive"] rerun_process = subprocess.Popen(cmd, env=env) time.sleep(0.5) # Wait for rerun server to start to remove a logged warning - cmd = [python_executable, example, "--connect", "--addr", f"127.0.0.1:{PORT}"] + extra_args + cmd = ["python", example, "--connect", "--addr", f"127.0.0.1:{PORT}"] + extra_args python_process = subprocess.Popen(cmd, env=env) print("Waiting for python process to finish…") diff --git a/tests/roundtrips.py b/tests/roundtrips.py index 66730d0ac01a..9d62cb1bcc90 100755 --- a/tests/roundtrips.py +++ b/tests/roundtrips.py @@ -205,7 +205,8 @@ def run_roundtrip_cpp(arch: str, release: bool) -> str: cmake_build(target_name, release) - cmd = [f"{cpp_build_dir}/tests/cpp/roundtrips/{target_name}", output_path] + target_path = f"Release/{target_name}.exe" if os.name == "nt" else target_name + cmd = [f"{cpp_build_dir}/tests/cpp/roundtrips/{target_path}", output_path] run(cmd, env=roundtrip_env(), timeout=12000) return output_path