diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf155e7..1f8e663 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,47 +39,63 @@ jobs: # Verify verify: - name: "Check ${{ matrix.chip }}" - runs-on: ubuntu-latest + runs-on: macos-m1-self-hosted - strategy: - fail-fast: false - matrix: - chip: [esp32, esp32c2, esp32c3, esp32c6, esp32h2, esp32s2, esp32s3] + # chip: [esp32, esp32c2, esp32c3, esp32c6, esp32h2, esp32s2, esp32s3] steps: - uses: actions/checkout@v4 - # Rust toolchain for Xtensa: - - if: ${{ contains(fromJson('["esp32", "esp32s2", "esp32s3"]'), matrix.chip) }} - uses: esp-rs/xtensa-toolchain@v1.5 + # # Rust toolchain for Xtensa: + # - if: ${{ contains(fromJson('["esp32", "esp32s2", "esp32s3"]'), matrix.chip) }} + # uses: esp-rs/xtensa-toolchain@v1.5 + # with: + # default: true + # buildtargets: ${{ matrix.chip }} + # ldproxy: false + + # # Rust toolchain for RISC-V: + # - if: ${{ !contains(fromJson('["esp32", "esp32s2", "esp32s3"]'), matrix.chip) }} + # uses: dtolnay/rust-toolchain@stable + # with: + # target: riscv32imc-unknown-none-elf,riscv32imac-unknown-none-elf + # components: clippy,rustfmt,rust-src + + # # Rust toolchain for RISC-V: + # - if: ${{ !contains(fromJson('["esp32", "esp32s2", "esp32s3"]'), matrix.chip) }} + # uses: dtolnay/rust-toolchain@nightly + # with: + # target: riscv32imc-unknown-none-elf,riscv32imac-unknown-none-elf + # components: clippy,rustfmt,rust-src + + # Install the Rust toolchain for Xtensa devices: + - uses: esp-rs/xtensa-toolchain@v1.6 with: - default: true - buildtargets: ${{ matrix.chip }} - ldproxy: false + version: 1.90.0.0 - # Rust toolchain for RISC-V: - - if: ${{ !contains(fromJson('["esp32", "esp32s2", "esp32s3"]'), matrix.chip) }} - uses: dtolnay/rust-toolchain@stable + # Install the Rust stable toolchain for RISC-V devices: + - uses: dtolnay/rust-toolchain@v1 with: target: riscv32imc-unknown-none-elf,riscv32imac-unknown-none-elf - components: clippy,rustfmt,rust-src + toolchain: stable + components: rust-src - # Rust toolchain for RISC-V: - - if: ${{ !contains(fromJson('["esp32", "esp32s2", "esp32s3"]'), matrix.chip) }} - uses: dtolnay/rust-toolchain@nightly - with: - target: riscv32imc-unknown-none-elf,riscv32imac-unknown-none-elf - components: clippy,rustfmt,rust-src + # Prepare cargo-batch + - name: Setup cargo-batch + run: | + if ! command -v cargo-batch &> /dev/null; then + cargo install --git https://github.com/embassy-rs/cargo-batch cargo --bin cargo-batch --locked --force + fi - # //Define a new environment variable called toolchain - - if: ${{ contains(fromJson('["esp32", "esp32s2", "esp32s3"]'), matrix.chip) }} - run: echo "TOOLCHAIN=+esp" >> $GITHUB_ENV + # # //Define a new environment variable called toolchain + # - if: ${{ contains(fromJson('["esp32", "esp32s2", "esp32s3"]'), matrix.chip) }} + # run: echo "TOOLCHAIN=+esp" >> $GITHUB_ENV - uses: Swatinem/rust-cache@v2 - name: Generate and check project - run: cargo ${{ env.TOOLCHAIN }} xtask check ${{ matrix.chip }} ${{ fromJSON('["", "--all-combinations"]')[inputs.all_combinations || github.event_name == 'schedule'] }} ${{ fromJSON('["", "--build"]')[inputs.build || github.event_name == 'schedule'] }} + run: cargo xtask check-all + # run: cargo ${{ env.TOOLCHAIN }} xtask check ${{ matrix.chip }} --all-combinations --build - if: github.event_name == 'schedule' name: Run cargo-package diff --git a/template/.cargo/config.toml b/template/.cargo/config.toml index 2adc905..e218691 100644 --- a/template/.cargo/config.toml +++ b/template/.cargo/config.toml @@ -30,6 +30,13 @@ rustflags = [ #IF option("stack-smashing-protection") "-Z", "stack-protector=all", #ENDIF +#IF option("defmt") + "-C", "link-arg=-Tdefmt.x", +#ENDIF +#IF option("embedded-test") + "-C", "link-arg=-Tembedded-test.x", +#ENDIF + "-C", "link-arg=-Tlinkall.x", ] #REPLACE riscv32imac-unknown-none-elf rust_target diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 5ee1752..04cdb07 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -13,3 +13,4 @@ esp-generate = { path = "..", default-features = false } log = "0.4.28" tempfile = "3.23.0" serde_yaml = "0.9.33" +strum = { version = "0.27.1", features = ["derive"] } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 904a0ef..4ff9792 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,10 +1,12 @@ use std::{ - collections::HashSet, + collections::{HashSet, HashMap}, path::{Path, PathBuf}, process::{Command, Stdio}, }; -use anyhow::{bail, Result}; +use std::ffi::OsStr; + +use anyhow::{bail, Result, Context}; use clap::{Parser, Subcommand}; use esp_generate::{ config::{find_option, ActiveConfiguration}, @@ -13,6 +15,8 @@ use esp_generate::{ use esp_metadata::Chip; use log::info; +use strum::IntoEnumIterator as _; + // Unfortunate hard-coded list of non-codegen options const IGNORED_CATEGORIES: &[&str] = &["editor", "optional"]; @@ -40,8 +44,199 @@ enum Commands { #[arg(short, long)] dry_run: bool, }, + /// Technically equal to `Check` command executed with `--all-combinations --build` options for every chip. + /// Intented to be used for global testing using `cargo-batch` + CheckAll { + /// Just print what would be tested + #[arg(short, long)] + dry_run: bool, + }, +} + +/// A builder for constructing cargo command line arguments. +#[derive(Clone, Debug, Default)] +pub struct CargoArgsBuilder { + pub(crate) artifact_name: String, + pub(crate) config_path: Option, + pub(crate) manifest_path: Option, + pub(crate) toolchain: Option, + pub(crate) subcommand: String, + pub(crate) target: Option, + pub(crate) features: Vec, + pub(crate) args: Vec, + pub(crate) configs: Vec, + pub(crate) env_vars: HashMap, + pub(crate) current_dir: Option, +} + +impl CargoArgsBuilder { + pub fn new(artifact_name: String) -> Self { + Self { + subcommand: artifact_name.clone(), + artifact_name, + ..Default::default() + } } + /// Set the path to the Cargo manifest file (Cargo.toml) + #[must_use] + pub fn manifest_path(mut self, path: PathBuf) -> Self { + self.manifest_path = Some(path); + self + } + + #[must_use] + pub fn current_dir>(mut self, dir: P) -> Self { + self.current_dir = Some(dir.into()); + self + } + + /// Set the path to the Cargo configuration file (.cargo/config.toml) + #[must_use] + pub fn config_path(mut self, path: PathBuf) -> Self { + self.config_path = Some(path); + self + } + + /// Set the Rust toolchain to use. + #[must_use] + pub fn toolchain(mut self, toolchain: S) -> Self + where + S: Into, + { + self.toolchain = Some(toolchain.into()); + self + } + + /// Set the cargo subcommand to use. + #[must_use] + pub fn subcommand(mut self, subcommand: S) -> Self + where + S: Into, + { + self.subcommand = subcommand.into(); + self + } + + /// Set the compilation target to use. + #[must_use] + pub fn target(mut self, target: S) -> Self + where + S: Into, + { + self.target = Some(target.into()); + self + } + + /// Set the cargo features to use. + #[must_use] + pub fn features(mut self, features: &[String]) -> Self { + self.features = features.to_vec(); + self + } + + /// Add a single argument to the cargo command line. + #[must_use] + pub fn arg(mut self, arg: S) -> Self + where + S: Into, + { + self.args.push(arg.into()); + self + } + + /// Add multiple arguments to the cargo command line. + #[must_use] + pub fn args(mut self, args: &[S]) -> Self + where + S: Clone + Into, + { + for arg in args { + self.args.push(arg.clone().into()); + } + self + } + + /// Add a single argument to the cargo command line. + pub fn add_arg(&mut self, arg: S) -> &mut Self + where + S: Into, + { + self.args.push(arg.into()); + self + } + + /// Adds a raw configuration argument (--config, -Z, ...) + #[must_use] + pub fn config(mut self, arg: S) -> Self + where + S: Into, + { + self.add_config(arg); + self + } + + /// Adds a raw configuration argument (--config, -Z, ...) + pub fn add_config(&mut self, arg: S) -> &mut Self + where + S: Into, + { + self.configs.push(arg.into()); + self + } + + /// Adds an environment variable + pub fn add_env_var(&mut self, key: S, value: S) -> &mut Self + where + S: Into, + { + self.env_vars.insert(key.into(), value.into()); + self + } + + /// Build the final list of cargo command line arguments. + #[must_use] + pub fn build(&self) -> Vec { + let mut args = vec![]; + + if let Some(ref toolchain) = self.toolchain { + args.push(format!("+{toolchain}")); + } + + args.push(self.subcommand.clone()); + + if let Some(manifest_path) = &self.manifest_path { + args.push("--manifest-path".to_string()); + args.push(manifest_path.display().to_string()); + } + + if let Some(config_path) = &self.config_path { + args.push("--config".to_string()); + args.push(config_path.display().to_string()); + } + + if let Some(ref target) = self.target { + args.push(format!("--target={target}")); + } + + for config in self.configs.iter() { + args.push(config.clone()); + } + + if !self.features.is_empty() { + args.push(format!("--features={}", self.features.join(","))); + } + + for arg in self.args.iter() { + args.push(arg.clone()); + } + + log::debug!("Built cargo args: {:?}", args); + args + } +} + + fn main() -> Result<()> { env_logger::Builder::new() .filter_module("xtask", log::LevelFilter::Info) @@ -59,12 +254,13 @@ fn main() -> Result<()> { build, dry_run, } => check(&workspace, chip, all_combinations, build, dry_run), + + Commands::CheckAll { + dry_run, + } => check_all(&workspace, dry_run), } } -// ---------------------------------------------------------------------------- -// CHECK - fn check( workspace: &Path, chip: Chip, @@ -102,57 +298,143 @@ fn check( // specified generation options: generate(workspace, &project_path, PROJECT_NAME, chip, &options)?; + let current_dir = project_path.join(PROJECT_NAME); + + // batcher **per project** + let mut commands = CargoCommandBatcher::new(); + // Ensure that the generated project builds without errors: - let output = Command::new("cargo") - .args([if build { "build" } else { "check" }]) - .env_remove("RUSTUP_TOOLCHAIN") - .current_dir(project_path.join(PROJECT_NAME)) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .output()?; - if !output.status.success() { - bail!("Failed to execute cargo check subcommand") - } + commands.push( + CargoArgsBuilder::new(if build { "build".to_string() } else { "check".to_string() }) + .current_dir(¤t_dir) + .target(chip.target()), + ); // Ensure that the generated test project builds also: if options.iter().any(|o| o == "embedded-test") { - let output = Command::new("cargo") - .args(["test", "--no-run"]) - .env_remove("RUSTUP_TOOLCHAIN") - .current_dir(project_path.join(PROJECT_NAME)) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .output()?; - if !output.status.success() { - bail!("Failed to execute cargo test subcommand") - } + commands.push( + CargoArgsBuilder::new("test".to_string()) + .args(&["--no-run".to_string()]) + .current_dir(¤t_dir) + .target(chip.target()), + ); } // Run clippy against the generated project to check for lint errors: - let output = Command::new("cargo") - .args(["clippy", "--no-deps", "--", "-Dwarnings"]) - .env_remove("RUSTUP_TOOLCHAIN") - .current_dir(project_path.join(PROJECT_NAME)) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .output()?; - if !output.status.success() { - bail!("Failed to execute cargo clippy subcommand") + // commands.push( + // CargoArgsBuilder::new("clippy".to_string()) + // .args(&["--no-deps".to_string(), "--".to_string(), "-Dwarnings".to_string()]) + // .current_dir(¤t_dir) + // .target(chip.target()), + // ); + + // TODO get me back + // commands.push(CargoArgsBuilder::new("fmt".to_string()) + // .args(&["--".to_string(), "--check".to_string()])); + + for c in commands.build(false) { + println!("Command: cargo {}", c.command.join(" ").replace("---", "\n ---")); + c.run()?; } + } + + Ok(()) +} + + +// Run a full check over every chip in one cargo-batch. +fn check_all( + workspace: &Path, + dry_run: bool, +) -> Result<()> { + if dry_run { + info!("Dry run — no commands executed."); + return Ok(()); + } + + let mut batch = CargoCommandBatcher::new(); + // Keep tempdirs alive until after the batch executes + let mut _tempdirs: Vec = Vec::new(); + + const PROJECT_NAME: &str = "test"; + + // let chip = Chip::Esp32; + + for chip in Chip::iter().collect::>() { + log::info!("BUILD: {chip}"); + + info!("Going to check"); + let to_check = options_for_chip(chip, true)?; + for check in &to_check { + info!("\"{}\"", check.join(", ")); + } + + for options in to_check { + log::info!("WITH OPTIONS: {options:?}"); + + // We will generate the project in a temporary directory, to avoid + // making a mess when this subcommand is executed locally: + let project_dir = tempfile::tempdir()?; + let project_path = project_dir.path().to_path_buf(); + log::info!("PROJECT PATH: {project_path:?}"); + + // Generate a project targeting the specified chip and using the + // specified generation options: + generate(workspace, &project_path, PROJECT_NAME, chip, &options)?; + + let project_root = project_path.join(PROJECT_NAME); + let manifest_path = project_root.join("Cargo.toml"); + let config_path = project_root.join(".cargo").join("config.toml"); + + // Hold the tempdir so it isn’t deleted before we run the batch + _tempdirs.push(project_dir); + + // let manifest_path = project_path.join(PROJECT_NAME).join("Cargo.toml"); + + // Ensure that the generated project builds without errors: + batch.push( + CargoArgsBuilder::new("build".to_string()) + .manifest_path(manifest_path.clone()) + .config_path(config_path.clone()) + // .current_dir(¤t_dir) + .target(chip.target()), + ); + + // Ensure that the generated test project builds also: + if options.iter().any(|o| o == "embedded-test") { + batch.push( + CargoArgsBuilder::new("test".to_string()) + .args(&["--no-run".to_string()]) + .manifest_path(manifest_path.clone()) + .config_path(config_path.clone()) + .target(chip.target()), + ); + } - // Ensure that the generated project is correctly formatted: - let output = Command::new("cargo") - .args(["fmt", "--", "--check"]) - .env_remove("RUSTUP_TOOLCHAIN") - .current_dir(project_path.join(PROJECT_NAME)) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .output()?; - if !output.status.success() { - bail!("Failed to execute cargo fmt subcommand") + // Run clippy against the generated project to check for lint errors: + // batch.push( + // CargoArgsBuilder::new("clippy".to_string()) + // .args(&["--no-deps".to_string(), "--".to_string(), "-Dwarnings".to_string()]) + // .manifest_path(manifest_path.clone()) + // .config_path(config_path.clone()) + // .target(chip.target()), + // ); + + // Ensure that the generated project is correctly formatted: + // batch.push( + // CargoArgsBuilder::new("fmt".to_string()) + // .args(&["--".to_string(), "--check".to_string()]) + // .manifest_path(manifest_path.clone()) + // .config_path(config_path.clone()) + // ); } } + for c in batch.build(false) { + println!("Command: cargo {}", c.command.join(" ").replace("---", "\n ---")); + c.run()?; + } + Ok(()) } @@ -338,3 +620,285 @@ fn generate( Ok(()) } + + +#[derive(Debug, PartialEq, Eq, Hash, Clone)] +struct BatchKey { + config_file: String, + toolchain: Option, + config: Vec, + env_vars: Vec<(String, String)>, +} + +impl BatchKey { + fn from_command(command: &CargoArgsBuilder) -> Self { + let config_file = if let Some(config_path) = &command.config_path { + std::fs::read_to_string(config_path).unwrap_or_default() + } else { + String::new() + }; + + Self { + toolchain: command.toolchain.clone(), + config: command.configs.clone(), + config_file, + env_vars: { + let mut env_vars = command + .env_vars + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect::>(); + + env_vars.sort(); + env_vars + }, + } + } +} + +#[derive(Debug)] +pub struct CargoCommandBatcher { + commands: HashMap>, +} + +#[derive(Debug, Clone)] +pub struct BuiltCommand { + pub artifact_name: String, + pub command: Vec, + pub env_vars: Vec<(String, String)>, + // pub cwd: PathBuf, +} + +impl BuiltCommand { + pub fn run(&self) -> Result { + let cwd = self + .command + .windows(2) + .find_map(|w| { + if w[0] == "--manifest-path" { + Path::new(&w[1]).parent().map(|p| p.to_path_buf()) + } else { + None + } + }).unwrap(); + + run_with_env(&self.command, &cwd, self.env_vars.clone()) + } +} + +fn run_with_env(args: &[String], cwd: &Path, envs: I) -> Result +where + I: IntoIterator + core::fmt::Debug, + K: AsRef, + V: AsRef, +{ + if !cwd.is_dir() { + bail!("The `cwd` argument MUST be a directory"); + } + + #[cfg(target_os = "windows")] + fn windows_safe_path(p: &Path) -> &Path { + if let Ok(stripped) = p.strip_prefix(r"\\?\") { + stripped + } else { + p + } + } + #[cfg(not(target_os = "windows"))] + fn windows_safe_path(p: &Path) -> &Path { + p + } + + let cwd = windows_safe_path(cwd); + + log::debug!( + "Running `cargo {}` in {:?} - Environment {:?}", + args.join(" "), + cwd, + envs + ); + + let mut command = Command::new("cargo"); + command + .args(args) + .current_dir(cwd) + .env_remove("RUSTUP_TOOLCHAIN") + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + + if args.iter().any(|a| a.starts_with('+')) { + command.env_remove("CARGO"); + } + + let output = command + .stdin(Stdio::inherit()) + .output() + .with_context(|| format!("Couldn't get output for command {command:?}"))?; + + if output.status.success() { + Ok(String::from_utf8_lossy(&output.stdout).to_string()) + } else { + bail!( + "Failed to execute cargo subcommand `cargo {}`", + args.join(" "), + ) + } +} + +impl CargoCommandBatcher { + pub fn new() -> Self { + Self { + commands: HashMap::new(), + } + } + + pub fn push(&mut self, command: CargoArgsBuilder) { + let key = BatchKey::from_command(&command); + self.commands.entry(key).or_default().push(command); + } + + fn build_for_cargo_batch(&self) -> Vec { + let mut all = Vec::new(); + + for (key, group) in self.commands.iter() { + if group.len() == 1 { + all.push(Self::build_one_for_cargo(&group[0])); + continue; + } + + let mut command = Vec::new(); + let mut batch_len = 0; + let mut commands_in_batch = 0; + + // Windows be Windows, it has a command length limit. + let limit = if cfg!(target_os = "windows") { + Some(8191) + } else { + None + }; + + for item in group.iter() { + // Only some commands can be batched + let batchable = ["build", "doc", "check"]; + if !batchable.iter().any(|&sub| sub == item.subcommand) { + all.push(Self::build_one_for_cargo(item)); + continue; + } + + let mut c = item.clone(); + c.toolchain = None; + c.configs = Vec::new(); + c.config_path = None; + + let args = c.build(); + let command_chars = 4 + args.iter().map(|arg| arg.len() + 1).sum::(); + + if !command.is_empty() + && let Some(limit) = limit + && batch_len + command_chars > limit + { + all.push(BuiltCommand { + artifact_name: String::from("batch"), + command: std::mem::take(&mut command), + env_vars: key.env_vars.clone(), + }); + } + + if command.is_empty() { + if let Some(tc) = key.toolchain.as_ref() { + command.push(format!("+{tc}")); + } + command.push(String::from("batch")); + if !key.config_file.is_empty() + && let Some(config_path) = &group[0].config_path + { + command.push("--config".to_string()); + command.push(config_path.display().to_string()); + } + command.extend_from_slice(&key.config); + + commands_in_batch = 0; + batch_len = command.iter().map(|s| s.len() + 1).sum::() - 1; + } + + command.push("---".to_string()); + command.extend_from_slice(&args); + + commands_in_batch += 1; + batch_len += command_chars; + } + + if commands_in_batch > 0 { + all.push(BuiltCommand { + artifact_name: String::from("batch"), + command, + env_vars: key.env_vars.clone(), + }); + } + } + + all + } + + + fn build_for_cargo(&self) -> Vec { + let mut all = Vec::new(); + + for group in self.commands.values() { + for item in group.iter() { + all.push(Self::build_one_for_cargo(item)); + } + } + + all + } + + pub fn build_one_for_cargo(item: &CargoArgsBuilder) -> BuiltCommand { + BuiltCommand { + artifact_name: item.artifact_name.clone(), + command: { + let mut args = item.build(); + + if item.args.iter().any(|arg| arg == "--artifact-dir") { + args.push("-Zunstable-options".to_string()); + } + + args + }, + env_vars: item + .env_vars + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + // cwd: item.current_dir.clone().expect("current_dir must be set"), + } + } + + pub fn build(&self, no_batch: bool) -> Vec { + let cargo_batch_available = Command::new("cargo") + .arg("batch") + .arg("-h") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false); + + if cargo_batch_available && !no_batch { + self.build_for_cargo_batch() + } else { + if !no_batch { + log::warn!("You don't have cargo batch installed. Falling back to cargo."); + log::warn!("You should really install cargo-batch."); + log::warn!( + "cargo install --git https://github.com/embassy-rs/cargo-batch cargo --bin cargo-batch --locked" + ); + } + self.build_for_cargo() + } + } +} + +impl Drop for CargoCommandBatcher { + fn drop(&mut self) {} +}