Merge rust scripts into re_dev_tools crate (rerun-io#5972)

### What

* Fixes rerun-io#5889

Puts a bunch of tooling under a new extensible home.

```
pixi run dev-tools --help
✨ Pixi task (default): cargo run -q --locked -p re_dev_tools -- --help
Usage: re_dev_tools <command> [<args>]

Various development tools for Rerun.

Options:
  --help            display usage information

Commands:
  build-examples    Build examples and their manifest.
  build-web-viewer  Build the web-viewer.
  search-index      Meilisearch indexer and REPL
```

Intentionally left out `re_types_builder` since it's fairly big on its
own.


### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using examples from latest `main` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/5972?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/5972?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG
* [x] If applicable, add a new check to the [release
checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)!

- [PR Build Summary](https://build.rerun.io/pr/5972)
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)

To run all checks from `main`, comment on the PR with `@rerun-bot
full-check`.

.github/workflows/reusable_build_and_upload_rerun_cli.yml

@@ -166,9 +166,7 @@ jobs:
           pixi-version: v0.19.0
 
       - name: Build web-viewer (release)
-        shell: bash
-        run: |
-          pixi run cargo run --locked -p re_build_web_viewer -- --release -g
+        run: pixi run rerun-build-web-release
 
       # This does not run in the pixi environment, doing so
       # causes it to select the wrong compiler on macos-arm64

.github/workflows/reusable_build_web.yml

@@ -71,7 +71,7 @@ jobs:
           if [ ${{ inputs.CHANNEL }} = "nightly" ]; then
             export DEFAULT_EXAMPLES_MANIFEST_URL="https://app.rerun.io/version/nightly/examples_manifest.json"
           fi
-          pixi run cargo run --locked -p re_build_web_viewer -- --release -g
+          pixi run rerun-build-web-release
 
       # We build a single manifest pointing to the `commit`
       # All the `pr`, `main`, release tag, etc. variants will always just point to the resolved commit

.github/workflows/reusable_checks_rust.yml

@@ -65,9 +65,7 @@ jobs:
       # We need to build the web viewer for `rust_checks.py` to succeed.
       # We build in release so that we can reuse the results for actual publishing, if necessary
       - name: Build web-viewer (release)
-        shell: bash
-        run: |
-          pixi run cargo run --locked -p re_build_web_viewer -- --release -g
+        run: pixi run rerun-build-web-release
 
       - name: Rust checks & tests
         if: ${{ inputs.CHANNEL == 'pr' }}

.github/workflows/reusable_publish_web.yml

@@ -108,7 +108,7 @@ jobs:
       - name: Build web-viewer (release)
         shell: bash
         run: |
-          pixi -e wheel-test run cargo run --locked -p re_build_web_viewer -- --release -g
+          pixi -e wheel-test run rerun-build-web-release
 
       - name: Build examples
         shell: bash

.github/workflows/reusable_release_crates.yml

@@ -29,7 +29,7 @@ jobs:
           pixi-version: v0.19.0
 
       - name: Build web-viewer (release)
-        run: pixi run cargo run --locked -p re_build_web_viewer -- --release -g
+        run: pixi run rerun-build-web-release
 
       - name: Publish
         run: pixi run python scripts/ci/crates.py publish --token ${{ secrets.CRATES_IO_TOKEN }}

ARCHITECTURE.md

@@ -176,14 +176,12 @@ Update instructions:
 
 ### Build support
 
-| Crate                      | Description                                                   |
-|----------------------------|---------------------------------------------------------------|
-| re_build_info              | Information about the build. Use together with re_build_tools |
-| re_build_tools             | build.rs helpers for generating build info                    |
-| re_types_builder           | Generates code for Rerun's SDKs from flatbuffers definitions. |
-| re_build_examples          | Build rerun example RRD files and manifest                    |
-| re_build_search_index      | Build a Meilisearch index from all our documentation          |
-| re_build_web_viewer        | Build the rerun web-viewer Wasm from source                   |
+| Crate                      | Description                                                      |
+|----------------------------|------------------------------------------------------------------|
+| re_build_info              | Information about the build. Use together with re_build_tools    |
+| re_build_tools             | build.rs helpers for generating build info                       |
+| re_types_builder           | Generates code for Rerun's SDKs from flatbuffers definitions.    |
+| re_dev_tools               | Various tools for Rerun development. Each tool has a subcommand. |
 
 
 ### Utilities

Cargo.toml

@@ -29,15 +29,13 @@ version = "0.16.0-alpha.1+dev"
 # In particular: if we compile rerun 0.3.0-alpha.0 we only want it to use
 # re_log_types 0.3.0-alpha.0, NOT 0.3.0-alpha.4 even though it is newer and semver-compatible.
 re_analytics = { path = "crates/re_analytics", version = "=0.16.0-alpha.1", default-features = false }
-re_build_search_index = { path = "crates/re_build_search_index", version = "=0.16.0-alpha.1", default-features = false }
-re_build_examples = { path = "crates/re_build_examples", version = "=0.16.0-alpha.1", default-features = false }
 re_build_info = { path = "crates/re_build_info", version = "=0.16.0-alpha.1", default-features = false }
 re_build_tools = { path = "crates/re_build_tools", version = "=0.16.0-alpha.1", default-features = false }
-re_build_web_viewer = { path = "crates/re_build_web_viewer", version = "=0.16.0-alpha.1", default-features = false }
 re_crash_handler = { path = "crates/re_crash_handler", version = "=0.16.0-alpha.1", default-features = false }
 re_data_source = { path = "crates/re_data_source", version = "=0.16.0-alpha.1", default-features = false }
 re_data_store = { path = "crates/re_data_store", version = "=0.16.0-alpha.1", default-features = false }
 re_data_ui = { path = "crates/re_data_ui", version = "=0.16.0-alpha.1", default-features = false }
+re_dev_tools = { path = "crates/re_dev_tools", version = "=0.16.0-alpha.1", default-features = false }
 re_entity_db = { path = "crates/re_entity_db", version = "=0.16.0-alpha.1", default-features = false }
 re_error = { path = "crates/re_error", version = "=0.16.0-alpha.1", default-features = false }
 re_format = { path = "crates/re_format", version = "=0.16.0-alpha.1", default-features = false }
@@ -47,9 +45,9 @@ re_log_encoding = { path = "crates/re_log_encoding", version = "=0.16.0-alpha.1"
 re_log_types = { path = "crates/re_log_types", version = "=0.16.0-alpha.1", default-features = false }
 re_memory = { path = "crates/re_memory", version = "=0.16.0-alpha.1", default-features = false }
 re_query = { path = "crates/re_query", version = "=0.16.0-alpha.1", default-features = false }
-re_query2 = { path = "crates/re_query2", version = "=0.16.0-alpha.1", default-features = false }
 re_query_cache = { path = "crates/re_query_cache", version = "=0.16.0-alpha.1", default-features = false }
 re_query_cache2 = { path = "crates/re_query_cache2", version = "=0.16.0-alpha.1", default-features = false }
+re_query2 = { path = "crates/re_query2", version = "=0.16.0-alpha.1", default-features = false }
 re_renderer = { path = "crates/re_renderer", version = "=0.16.0-alpha.1", default-features = false }
 re_sdk = { path = "crates/re_sdk", version = "=0.16.0-alpha.1", default-features = false }
 re_sdk_comms = { path = "crates/re_sdk_comms", version = "=0.16.0-alpha.1", default-features = false }

crates/re_build_search_index/Cargo.toml → crates/re_dev_tools/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
-name = "re_build_search_index"
+name = "re_dev_tools"
 authors.workspace = true
-description = "Build a Meilisearch index from all of Rerun's documentation"
+description = "Various tools for rerun development"
 edition.workspace = true
 homepage.workspace = true
 include.workspace = true
@@ -17,8 +17,8 @@ all-features = true
 
 
 [dependencies]
-re_build_examples.workspace = true
 re_build_tools.workspace = true
+re_format.workspace = true
 
 # External
 anyhow.workspace = true
@@ -28,8 +28,8 @@ cargo_metadata.workspace = true
 glob.workspace = true
 indicatif.workspace = true
 itertools.workspace = true
-roxmltree = "0.19.0"
 rayon.workspace = true
+roxmltree = "0.19.0"
 rustdoc-json = "0.8.8"
 rustdoc-types = "0.24.0"
 serde = { workspace = true, features = ["derive"] }
@@ -38,3 +38,4 @@ serde_yaml.workspace = true
 toml = { workspace = true, features = ["parse", "preserve_order"] }
 ureq = { workspace = true, features = ["json"] }
 url.workspace = true
+wasm-bindgen-cli-support.workspace = true

crates/re_dev_tools/README.md

@@ -0,0 +1,11 @@
+# re_dev_tools
+Crate that combines several development utilities.
+To get an overview over all tools run `pixi run dev-tools --help`.
+
+We keep all smaller rust "scripts" in this single crate so we don't needlessly
+increase the number of such utility crates and to make it easy to get
+an overview over all build tooling written in Rust.
+
+## Adding a new tool
+* Create a folder under `src` with your new tool
+* Add a new enum entry to `main.rs`