Centralized build tools in re_build_tools (rerun-io#2331)

Centralize our build tools in one place (`re_build_tools`) and re-use
those everywhere.

Cherry picked from rerun-io#2329 which both re-uses existing tools adds new
ones.

~Technically, `re_build_build_info` is not the correct place for this,
and one could imagine yet another crate `re_build_tools` for these
things; but for now the overhead of yet another crate doesn't seem worth
it.~

Cargo.toml

@@ -25,8 +25,8 @@ version = "0.7.0-alpha.0"
 # re_log_types 0.3.0-alpha.0, NOT 0.3.0-alpha.4 even though it is newer and semver-compatible.
 re_analytics = { path = "crates/re_analytics", version = "0.7.0-alpha.0", default-features = false }
 re_arrow_store = { path = "crates/re_arrow_store", version = "0.7.0-alpha.0", default-features = false }
-re_build_build_info = { path = "crates/re_build_build_info", version = "0.7.0-alpha.0", default-features = false }
 re_build_info = { path = "crates/re_build_info", version = "0.7.0-alpha.0", default-features = false }
+re_build_tools = { path = "crates/re_build_tools", version = "0.7.0-alpha.0", default-features = false }
 re_build_web_viewer = { path = "crates/re_build_web_viewer", version = "0.7.0-alpha.0", default-features = false }
 re_components = { path = "crates/re_components", version = "0.7.0-alpha.0", default-features = false }
 re_crash_handler = { path = "crates/re_crash_handler", version = "0.7.0-alpha.0", default-features = false }

crates/re_analytics/Cargo.toml

@@ -43,4 +43,4 @@ web-sys = { version = "0.3.58", features = ["Storage"] }
 
 
 [build-dependencies]
-re_build_build_info.workspace = true
+re_build_tools.workspace = true

crates/re_analytics/build.rs

@@ -1,4 +1,4 @@
 fn main() {
-    re_build_build_info::rebuild_if_crate_changed("re_analytics");
-    re_build_build_info::export_env_vars();
+    re_build_tools::rebuild_if_crate_changed("re_analytics");
+    re_build_tools::export_env_vars();
 }

crates/re_build_info/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "re_build_info"
 authors.workspace = true
-description = "Information about the build. Use together with re_build_build_info"
+description = "Information about the build. Use together with re_build_tools"
 edition.workspace = true
 homepage.workspace = true
 include.workspace = true

crates/re_build_info/src/lib.rs

@@ -1,6 +1,6 @@
 //! Information about the build of a Rust crate.
 //!
-//! To use this you also need to call `re_build_build_info::export_env_vars()` from your build.rs.
+//! To use this you also need to call `re_build_tools::export_env_vars()` from your build.rs.
 
 mod build_info;
 mod crate_version;
@@ -9,7 +9,7 @@ pub use build_info::BuildInfo;
 pub use crate_version::CrateVersion;
 
 /// Create a [`BuildInfo`] at compile-time using environment variables exported by
-/// calling `re_build_build_info::export_env_vars()` from your build.rs.
+/// calling `re_build_tools::export_env_vars()` from your build.rs.
 #[macro_export]
 macro_rules! build_info {
     () => {

crates/re_build_build_info/Cargo.toml → crates/re_build_tools/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "re_build_build_info"
+name = "re_build_tools"
 authors.workspace = true
 description = "build.rs helpers for generating build info"
 edition.workspace = true

crates/re_build_build_info/README.md → crates/re_build_tools/README.md

@@ -1,9 +1,9 @@
-# re_build_build_info
+# re_build_tools
 
 Part of the [`rerun`](https://github.com/rerun-io/rerun) family of crates.
 
-[![Latest version](https://img.shields.io/crates/v/re_build_build_info.svg)](https://crates.io/crates/re_build_build_info)
-[![Documentation](https://docs.rs/re_build_build_info/badge.svg)](https://docs.rs/re_build_build_info)
+[![Latest version](https://img.shields.io/crates/v/re_build_tools.svg)](https://crates.io/crates/re_build_tools)
+[![Documentation](https://docs.rs/re_build_tools/badge.svg)](https://docs.rs/re_build_tools)
 ![MIT](https://img.shields.io/badge/license-MIT-blue.svg)
 ![Apache](https://img.shields.io/badge/license-Apache-blue.svg)
 

crates/re_build_build_info/src/lib.rs → crates/re_build_tools/src/lib.rs

@@ -10,7 +10,10 @@ use std::process::Command;
 
 mod rebuild_detector;
 
-pub use rebuild_detector::rebuild_if_crate_changed;
+pub use rebuild_detector::{
+    get_and_track_env_var, is_tracked_env_var_set, rebuild_if_crate_changed, rerun_if_changed,
+    rerun_if_changed_glob, rerun_if_changed_or_doesnt_exist, write_file_if_necessary,
+};
 
 // Situations to consider
 // ----------------------
@@ -74,22 +77,13 @@ pub fn export_env_vars() {
         if let Ok(head) = std::fs::read_to_string(&head_path) {
             if let Some(git_file) = head.strip_prefix("ref: ") {
                 if let Ok(path) = git_path(git_file) {
-                    rerun_if_changed(&path); // Track changes to commit hash
+                    rerun_if_changed(path); // Track changes to commit hash
                 }
             }
         }
     }
 }
 
-fn rerun_if_changed(path: &str) {
-    // Make sure the file exists, otherwise we'll be rebuilding all the time.
-    assert!(
-        std::path::Path::new(path).exists(),
-        "Failed to find {path:?}"
-    );
-    println!("cargo:rerun-if-changed={path}");
-}
-
 fn set_env(name: &str, value: &str) {
     println!("cargo:rustc-env={name}={value}");
 }

crates/re_build_build_info/src/rebuild_detector.rs → crates/re_build_tools/src/rebuild_detector.rs

@@ -2,7 +2,7 @@
 
 use std::{
     collections::{HashMap, HashSet},
-    path::PathBuf,
+    path::{Path, PathBuf},
 };
 
 use cargo_metadata::{CargoOpt, Metadata, MetadataCommand, Package, PackageId};
@@ -38,34 +38,71 @@ pub fn rebuild_if_crate_changed(pkg_name: &str) {
     }
 }
 
-fn get_and_track_env_var(env_var_name: &str) -> Result<String, std::env::VarError> {
+/// Call from `build.rs` to trigger a rebuild whenever an environment variable changes.
+pub fn get_and_track_env_var(env_var_name: &str) -> Result<String, std::env::VarError> {
     println!("cargo:rerun-if-env-changed={env_var_name}");
     std::env::var(env_var_name)
 }
 
-fn is_tracked_env_var_set(env_var_name: &str) -> bool {
+/// Call from `build.rs` to trigger a rebuild whenever an environment variable changes, and returns
+/// true if that variable has been set to a truthy value.
+pub fn is_tracked_env_var_set(env_var_name: &str) -> bool {
     let var = get_and_track_env_var(env_var_name).map(|v| v.to_lowercase());
     var == Ok("1".to_owned()) || var == Ok("yes".to_owned()) || var == Ok("true".to_owned())
 }
 
-fn rerun_if_changed(path: &std::path::Path) {
+/// Call from `build.rs` to trigger a rebuild whenever the file at `path` changes.
+///
+/// This requires the file to exist, which may or may not be what you want!
+pub fn rerun_if_changed(path: impl AsRef<Path>) {
+    let path = path.as_ref();
     // Make sure the file exists, otherwise we'll be rebuilding all the time.
     assert!(path.exists(), "Failed to find {path:?}");
     println!("cargo:rerun-if-changed={}", path.to_str().unwrap());
 }
 
-fn rerun_if_changed_glob(path: &str, files_to_watch: &mut HashSet<PathBuf>) {
+/// Call from `build.rs` to trigger a rebuild whenever the file at `path` changes, or it doesn't
+/// exist.
+pub fn rerun_if_changed_or_doesnt_exist(path: impl AsRef<Path>) {
+    let path = path.as_ref();
+    println!("cargo:rerun-if-changed={}", path.to_str().unwrap());
+}
+
+/// Call from `build.rs` to trigger a rebuild whenever any of the files identified by the given
+/// globbed `path` change.
+pub fn rerun_if_changed_glob(path: impl AsRef<Path>, files_to_watch: &mut HashSet<PathBuf>) {
+    let path = path.as_ref();
+
     // Workaround for windows verbatim paths not working with glob.
     // Issue: https://github.com/rust-lang/glob/issues/111
     // Fix: https://github.com/rust-lang/glob/pull/112
     // Fixed on upstream, but no release containing the fix as of writing.
-    let path = path.trim_start_matches(r"\\?\");
+    let path = path.to_str().unwrap().trim_start_matches(r"\\?\");
 
     for path in glob::glob(path).unwrap() {
         files_to_watch.insert(path.unwrap());
     }
 }
 
+/// Writes `content` to a file iff it differs from what's already there.
+///
+/// This prevents recursive feedback loops where one generates source files from build.rs, which in
+/// turn triggers `cargo`'s implicit `rerun-if-changed=src/**` clause.
+//
+// TODO(cmc): use the same source tracking system as re_types* instead
+pub fn write_file_if_necessary(
+    path: impl AsRef<std::path::Path>,
+    content: &[u8],
+) -> std::io::Result<()> {
+    if let Ok(cur_bytes) = std::fs::read(&path) {
+        if cur_bytes == content {
+            return Ok(());
+        }
+    }
+
+    std::fs::write(path, content)
+}
+
 // ---
 
 struct Packages<'a> {
@@ -102,9 +139,9 @@ impl<'a> Packages<'a> {
 
             // NOTE: Since we track the cargo manifest, past this point we only need to
             // account for locally patched dependencies.
-            rerun_if_changed_glob(path.join("Cargo.toml").as_ref(), files_to_watch);
-            rerun_if_changed_glob(path.join("**/*.rs").as_ref(), files_to_watch);
-            rerun_if_changed_glob(path.join("**/*.wgsl").as_ref(), files_to_watch);
+            rerun_if_changed_glob(path.join("Cargo.toml"), files_to_watch);
+            rerun_if_changed_glob(path.join("**/*.rs"), files_to_watch);
+            rerun_if_changed_glob(path.join("**/*.wgsl"), files_to_watch);
         }
 
         // Track all direct and indirect dependencies of that root package
@@ -133,8 +170,8 @@ impl<'a> Packages<'a> {
                 let mut dep_path = dep_pkg.manifest_path.clone();
                 dep_path.pop();
 
-                rerun_if_changed_glob(dep_path.join("Cargo.toml").as_ref(), files_to_watch); // manifest too!
-                rerun_if_changed_glob(dep_path.join("**/*.rs").as_ref(), files_to_watch);
+                rerun_if_changed_glob(dep_path.join("Cargo.toml"), files_to_watch); // manifest too!
+                rerun_if_changed_glob(dep_path.join("**/*.rs"), files_to_watch);
             }
 
             if tracked.insert(dep_pkg.id.clone()) {

crates/re_renderer/Cargo.toml

@@ -116,6 +116,11 @@ web-sys = { version = "0.3.61", features = [
 
 # For build.rs:
 [build-dependencies]
+
+# Rerun
+re_build_tools.workspace = true
+
+# External
 anyhow.workspace = true
 clean-path = "0.2"
 pathdiff = "0.2"

crates/re_renderer/build.rs

@@ -14,24 +14,15 @@
 
 #![allow(clippy::unwrap_used)]
 
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
+use anyhow::{bail, ensure, Context as _};
 use walkdir::{DirEntry, WalkDir};
 
-// ---
-
-fn rerun_if_changed(path: &std::path::Path) {
-    // Make sure the file exists, otherwise we'll be rebuilding all the time.
-    assert!(path.exists(), "Failed to find {path:?}");
-    println!("cargo:rerun-if-changed={}", path.to_str().unwrap());
-}
+use re_build_tools::{is_tracked_env_var_set, rerun_if_changed, write_file_if_necessary};
 
 // ---
 
-use std::path::PathBuf;
-
-use anyhow::{bail, ensure, Context as _};
-
 /// A pre-parsed import clause, as in `#import <something>`.
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ImportClause {
@@ -118,11 +109,11 @@ fn main() {
         // repository.
         return;
     }
-    if std::env::var("IS_IN_RERUN_WORKSPACE") != Ok("yes".to_owned()) {
+    if !is_tracked_env_var_set("IS_IN_RERUN_WORKSPACE") {
         // Only run if we are in the rerun workspace, not on users machines.
         return;
     }
-    if std::env::var("RERUN_IS_PUBLISHING") == Ok("yes".to_owned()) {
+    if is_tracked_env_var_set("RERUN_IS_PUBLISHING") {
         // We don't need to rebuild - we should have done so beforehand!
         // See `RELEASES.md`
         return;
@@ -241,17 +232,3 @@ pub fn init() {
 
     write_file_if_necessary(file_path, contents.as_bytes()).unwrap();
 }
-
-/// Only touch the file if the contents has actually changed
-fn write_file_if_necessary(
-    dst_path: impl AsRef<std::path::Path>,
-    content: &[u8],
-) -> std::io::Result<()> {
-    if let Ok(cur_bytes) = std::fs::read(&dst_path) {
-        if cur_bytes == content {
-            return Ok(());
-        }
-    }
-
-    std::fs::write(dst_path, content)
-}