From cc2da4a2a4148e6059b8dc02e8fa4bbd2026fc3a Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Thu, 12 Dec 2024 15:10:46 -0500 Subject: [PATCH] ci: download gen_init_cpio with authentication The raw endpoint[0] now[1] seems to return HTTP 429 on the first request and unauthenticated API limits are exceeded after one request (so one of ubuntu/macos builders fails). Hopefully this works with authentication. Link: https://raw.githubusercontent.com/torvalds/linux/refs/heads/master/usr/gen_init_cpio.c [0] Link: https://github.com/orgs/community/discussions/146957 [1] --- .github/workflows/ci.yml | 5 ++- Cargo.toml | 2 + xtask/Cargo.toml | 3 ++ xtask/src/run.rs | 81 +++++++++++++++++++++++++++------------- 4 files changed, 64 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 868d1f67a..c4d4d6a24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -299,7 +299,10 @@ jobs: run: cargo xtask integration-test local - name: Run virtualized integration tests - run: find test/.tmp -name 'vmlinuz-*' | xargs -t cargo xtask integration-test vm + run: | + set -euxo pipefail + find test/.tmp -name 'vmlinuz-*' -print0 | xargs -t -0 \ + cargo xtask integration-test vm --github-api-token ${{ secrets.GITHUB_TOKEN }} # Provides a single status check for the entire build workflow. # This is used for merge automation, like Mergify, since GH actions diff --git a/Cargo.toml b/Cargo.toml index 51be0307b..f49bf6d6f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,6 +59,7 @@ edition = "2021" # them to do that, but in the meantime we need to be careful. [workspace.dependencies] anyhow = { version = "1", default-features = false } +base64 = { version = "0.22.1", default-features = false } assert_matches = { version = "1.5.0", default-features = false } async-io = { version = "2.0", default-features = false } bindgen = { version = "0.71", default-features = false } @@ -80,6 +81,7 @@ netns-rs = { version = "0.1", default-features = false } nix = { version = "0.29.0", default-features = false } num_enum = { version = "0.7", default-features = false } object = { version = "0.36", default-features = false } +octorust = { version = "0.7.0", default-features = false } once_cell = { version = "1.20.1", default-features = false } proc-macro2 = { version = "1", default-features = false } proc-macro2-diagnostics = { version = "0.10.1", default-features = false } diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 0c617deae..db4a28548 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -11,11 +11,13 @@ edition.workspace = true [dependencies] anyhow = { workspace = true, features = ["std"] } aya-tool = { path = "../aya-tool", version = "0.1.0", default-features = false } +base64 = { workspace = true, features = ["std"] } cargo_metadata = { workspace = true } clap = { workspace = true, features = ["derive"] } dialoguer = { workspace = true } diff = { workspace = true } indoc = { workspace = true } +octorust = { workspace = true, features = ["rustls-tls"] } proc-macro2 = { workspace = true } public-api = { workspace = true } quote = { workspace = true } @@ -23,4 +25,5 @@ rustdoc-json = { workspace = true } rustup-toolchain = { workspace = true } syn = { workspace = true } tempfile = { workspace = true } +tokio = { workspace = true, features = ["rt"] } which = { workspace = true } diff --git a/xtask/src/run.rs b/xtask/src/run.rs index 98f55c613..ddd003c68 100644 --- a/xtask/src/run.rs +++ b/xtask/src/run.rs @@ -10,9 +10,10 @@ use std::{ }; use anyhow::{anyhow, bail, Context as _, Result}; +use base64::engine::Engine as _; use cargo_metadata::{Artifact, CompilerMessage, Message, Target}; use clap::Parser; -use xtask::{exec, Errors, AYA_BUILD_INTEGRATION_BPF}; +use xtask::{Errors, AYA_BUILD_INTEGRATION_BPF}; #[derive(Parser)] enum Environment { @@ -24,6 +25,12 @@ enum Environment { }, /// Runs the integration tests in a VM. VM { + /// The Github API token to use if network requests to Github are made. + /// + /// This may be required if Github rate limits are exceeded. + #[clap(long)] + github_api_token: Option, + /// The kernel images to use. /// /// You can download some images with: @@ -167,7 +174,10 @@ pub fn run(opts: Options) -> Result<()> { Err(anyhow!("failures:\n{}", failures)) } } - Environment::VM { kernel_image } => { + Environment::VM { + github_api_token, + kernel_image, + } => { // The user has asked us to run the tests on a VM. This is involved; strap in. // // We need tools to build the initramfs; we use gen_init_cpio from the Linux repository, @@ -192,37 +202,56 @@ pub fn run(opts: Options) -> Result<()> { .try_exists() .context("failed to check existence of gen_init_cpio")? { - let mut curl = Command::new("curl"); - curl.args([ - "-sfSL", - "https://raw.githubusercontent.com/torvalds/linux/master/usr/gen_init_cpio.c", - ]); - let mut curl_child = curl + let client = octorust::Client::new( + String::from("aya-xtask-integration-test-run"), + github_api_token.map(octorust::auth::Credentials::Token), + )?; + let octorust::Response { + status: _, + headers: _, + body: octorust::types::ContentFile { mut content, .. }, + } = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(client.repos().get_content_file( + "torvalds", + "linux", + "usr/gen_init_cpio.c", + "master", + )) + .context("failed to download gen_init_cpio.c")?; + // Github very helpfully wraps their base64 at 10 columns /s. + content.retain(|c| !c.is_whitespace()); + let content = base64::engine::general_purpose::STANDARD + .decode(content) + .context("failed to decode gen_init_cpio.c")?; + + let mut clang = Command::new("clang"); + clang + .args(["-g", "-O2", "-x", "c", "-", "-o"]) + .arg(&gen_init_cpio); + let mut child = clang + .stdin(Stdio::piped()) .stdout(Stdio::piped()) + .stderr(Stdio::piped()) .spawn() - .with_context(|| format!("failed to spawn {curl:?}"))?; - let Child { stdout, .. } = &mut curl_child; - let curl_stdout = stdout.take().unwrap(); + .with_context(|| format!("failed to spawn {clang:?}"))?; - let mut clang = Command::new("clang"); - let clang = exec( - clang - .args(["-g", "-O2", "-x", "c", "-", "-o"]) - .arg(&gen_init_cpio) - .stdin(curl_stdout), - ); - - let output = curl_child + let Child { stdin, .. } = &mut child; + let mut stdin = stdin.take().unwrap(); + stdin + .write_all(&content) + .with_context(|| format!("failed to write to {clang:?} stdin"))?; + std::mem::drop(stdin); // Send EOF. + + let output = child .wait_with_output() - .with_context(|| format!("failed to wait for {curl:?}"))?; + .with_context(|| format!("failed to wait for {clang:?}"))?; let Output { status, .. } = &output; if status.code() != Some(0) { - bail!("{curl:?} failed: {output:?}") + bail!("{clang:?} failed: {output:?}") } - - // Check the result of clang *after* checking curl; in case the download failed, - // only curl's output will be useful. - clang?; } let mut errors = Vec::new();