From 5e25a5a44093d9963e1041b16a1d75cab305f9ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Gardstr=C3=B6m?= Date: Mon, 3 Mar 2025 20:57:12 +0100 Subject: [PATCH 1/4] aa --- docker/qemu.sh | 8 +++++++- src/docker/image.rs | 2 +- src/lib.rs | 2 +- targets.toml | 3 +++ xtask/src/build_docker_image.rs | 28 ++++++++++++++++++++++++++-- xtask/src/ci/target_matrix.rs | 4 ++-- xtask/src/codegen.rs | 5 +---- xtask/src/util.rs | 15 ++++++++++----- 8 files changed, 51 insertions(+), 16 deletions(-) diff --git a/docker/qemu.sh b/docker/qemu.sh index 3a144ea3d..d364983ac 100755 --- a/docker/qemu.sh +++ b/docker/qemu.sh @@ -131,7 +131,7 @@ build_static_slirp() { rm -rf "${td}" } - +# passed arg 1: $1 = arch, so for example "arm64" or "aarch64" main() { local version=5.1.0 @@ -140,6 +140,12 @@ main() { local arch="${1}" \ softmmu="${2:-}" + # if arch is what we're currently running, we can just ignore qemu. + if [ "$(uname -m)" = "${arch}" ]; then + echo "Already running on ${arch}, skipping QEMU installation." + exit 0 + fi + install_packages \ autoconf \ automake \ diff --git a/src/docker/image.rs b/src/docker/image.rs index 6749fe86d..5a7331da2 100644 --- a/src/docker/image.rs +++ b/src/docker/image.rs @@ -162,7 +162,7 @@ impl From for ImageReference { /// The architecture/platform to use in the image /// /// -#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Deserialize)] #[serde(try_from = "String")] pub struct ImagePlatform { /// CPU architecture, x86_64, aarch64 etc diff --git a/src/lib.rs b/src/lib.rs index a65668e17..171cb661b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -72,7 +72,7 @@ pub use self::rustc::{TargetList, VersionMetaExt}; pub const CROSS_LABEL_DOMAIN: &str = "org.cross-rs"; #[allow(non_camel_case_types)] -#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Hash, PartialOrd, Ord)] #[serde(from = "&str", into = "String")] #[serde(rename_all = "snake_case")] pub enum TargetTriple { diff --git a/targets.toml b/targets.toml index 759e6867a..ae7d46e38 100644 --- a/targets.toml +++ b/targets.toml @@ -391,6 +391,7 @@ run = true [[target]] target = "x86_64-pc-windows-gnu" os = "ubuntu-latest" +platforms = ["x86_64-unknown-linux-gnu"] cpp = true std = true run = true @@ -398,6 +399,7 @@ run = true [[target]] target = "i686-pc-windows-gnu" os = "ubuntu-latest" +platforms = ["x86_64-unknown-linux-gnu"] cpp = true std = true run = true @@ -530,6 +532,7 @@ special = true [[target]] target = "aarch64-unknown-linux-gnu" +platforms = ["x86_64-unknown-linux-gnu"] sub = "centos" os = "ubuntu-latest" cpp = true diff --git a/xtask/src/build_docker_image.rs b/xtask/src/build_docker_image.rs index c747aba50..20aac30e9 100644 --- a/xtask/src/build_docker_image.rs +++ b/xtask/src/build_docker_image.rs @@ -146,6 +146,12 @@ pub fn build_docker_image( .contents_first(true) .into_iter() .filter_map(|e| e.ok().filter(|f| f.file_type().is_file())) + // don't add native + .filter(|f| { + !f.file_name() + .to_string_lossy() + .starts_with("Dockerfile.native") + }) .filter_map(|f| { f.file_name() .to_string_lossy() @@ -156,6 +162,17 @@ pub fn build_docker_image( .collect(); } } + + // grab platform from the targets.toml file, if it exists. + targets.iter_mut().for_each(|t| { + if let Some(m) = get_matrix() + .iter() + .find(|m| m.target == t.name && m.sub == t.sub) + { + t.platform = m.platforms.clone(); + } + }); + let gha = std::env::var("GITHUB_ACTIONS").is_ok(); let mut progress = progress.map(|x| x.parse().unwrap()); if gha { @@ -178,10 +195,17 @@ pub fn build_docker_image( let mut results = vec![]; for (platform, (target, dockerfile)) in targets .iter() - .flat_map(|t| platforms.iter().map(move |p| (p, t))) + .flat_map(|(t, d)| { + t.platform + .as_deref() + .unwrap_or(&platforms) + .iter() + .map(move |p| (p, (t, d))) + }) + .filter(|(p, _)| platforms.contains(p)) { if gha && targets.len() > 1 { - gha_print("::group::Build {target}"); + gha_print(&format!("::group::Build {target} for {}", platform.target)); } else { msg_info.note(format_args!("Build {target} for {}", platform.target))?; } diff --git a/xtask/src/ci/target_matrix.rs b/xtask/src/ci/target_matrix.rs index 4b8a518c3..a9c93e31a 100644 --- a/xtask/src/ci/target_matrix.rs +++ b/xtask/src/ci/target_matrix.rs @@ -2,7 +2,7 @@ use std::process::Command; use clap::builder::{BoolishValueParser, PossibleValuesParser}; use clap::Parser; -use cross::{shell::Verbosity, CommandExt}; +use cross::{docker::ImagePlatform, shell::Verbosity, CommandExt}; use serde::{Deserialize, Serialize}; use crate::util::{get_matrix, gha_output, gha_print, CiTarget, ImageTarget}; @@ -244,7 +244,7 @@ fn process_try_comment(message: &str) -> cross::Result<(bool, TargetMatrixArgs)> #[serde(rename_all = "kebab-case")] struct TargetMatrixElement<'a> { pretty: String, - platforms: &'a [String], + platforms: &'a [ImagePlatform], target: &'a str, #[serde(skip_serializing_if = "Option::is_none")] sub: Option<&'a str>, diff --git a/xtask/src/codegen.rs b/xtask/src/codegen.rs index 1fe5217b9..eafaed4e7 100644 --- a/xtask/src/codegen.rs +++ b/xtask/src/codegen.rs @@ -39,10 +39,7 @@ pub static PROVIDED_IMAGES: &[ProvidedImage] = &["#, platform = &image_target .platforms() .iter() - .map(|p| { - let image_platform: ImagePlatform = - p.parse().expect("should be a valid platform"); - + .map(|image_platform| { image_platform .to_codegen_string() .expect("should be a valid platform") diff --git a/xtask/src/util.rs b/xtask/src/util.rs index 04f1b8cec..dea687a34 100644 --- a/xtask/src/util.rs +++ b/xtask/src/util.rs @@ -4,7 +4,7 @@ use std::io::Write; use std::path::{Path, PathBuf}; use std::process::Command; -use cross::shell::MessageInfo; +use cross::{docker::ImagePlatform, shell::MessageInfo}; use cross::{docker, CommandExt, ToUtf8}; use once_cell::sync::{Lazy, OnceCell}; @@ -46,9 +46,9 @@ pub struct CiTarget { /// if `true` publish the generated binaries for cross #[serde(default)] pub deploy: Option, - /// the platform to build this image for, defaults to `["linux/amd64"]`, takes multiple + /// the platform to build this image for, defaults to whatever is needed, takes multiple #[serde(skip_serializing_if = "Option::is_none")] - platforms: Option>, + pub platforms: Option>, /// if `true` signal that this target requires `-Zbuild-std` #[serde(skip_serializing_if = "Option::is_none")] pub build_std: Option, @@ -82,6 +82,8 @@ impl CiTarget { crate::ImageTarget { name: self.target.clone(), sub: self.sub.clone(), + // XXX: This does not align with platforms() by design, as we want to be able to know if the field was set or not. + platform: self.platforms.clone(), } } @@ -89,8 +91,8 @@ impl CiTarget { self.os == "ubuntu-latest" } - pub fn platforms(&self) -> &[String] { - self.platforms.as_ref().unwrap_or(&DEFAULT_PLATFORMS_STRING) + pub fn platforms(&self) -> &[ImagePlatform] { + self.platforms.as_deref().unwrap_or(DEFAULT_PLATFORMS) } } @@ -162,6 +164,7 @@ pub fn pull_image( pub struct ImageTarget { pub name: String, pub sub: Option, + pub platform: Option>, } impl ImageTarget { @@ -235,6 +238,7 @@ impl std::str::FromStr for ImageTarget { return Ok(ImageTarget { name: target.to_string(), sub: Some(sub.to_string()), + platform: None, }); } } @@ -242,6 +246,7 @@ impl std::str::FromStr for ImageTarget { Ok(ImageTarget { name: s.to_string(), sub: None, + platform: None, }) } } From 989e06b71cac91572cc5a7fb8f556c94aa747b21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Gardstr=C3=B6m?= Date: Mon, 3 Mar 2025 21:35:54 +0100 Subject: [PATCH 2/4] doubt this works, very WIP --- .github/workflows/ci.yml | 4 ++-- targets.toml | 4 +++- xtask/src/ci/target_matrix.rs | 20 ++++++++++++++++++++ xtask/src/codegen.rs | 1 - xtask/src/util.rs | 6 +++++- 5 files changed, 30 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d3f498f8..d348c05ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -186,7 +186,7 @@ jobs: id: build-docker-image if: steps.prepare-meta.outputs.has-image timeout-minutes: 120 - run: cargo xtask build-docker-image -v "${TARGET}${SUB:+.$SUB}" ${{ matrix.verbose && '-v' || '' }} + run: cargo xtask build-docker-image -v "${TARGET}${SUB:+.$SUB}" ${{ matrix.verbose && '-v' || '' }}${{ matrix.platform && '--platform ' }}${{ join(matrix.platform) || '' }} env: TARGET: ${{ matrix.target }} SUB: ${{ matrix.sub }} @@ -246,7 +246,7 @@ jobs: github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || startsWith(github.ref, 'refs/tags/v') ) - run: cargo xtask build-docker-image -v --push "${TARGET}${SUB:+.$SUB}" + run: cargo xtask build-docker-image -v --push "${TARGET}${SUB:+.$SUB}" ${{ matrix.verbose && '-v' || '' }}${{ matrix.platform && '--platform ' }}${{ join(matrix.platform) || '' }} env: TARGET: ${{ matrix.target }} SUB: ${{ matrix.sub }} diff --git a/targets.toml b/targets.toml index ae7d46e38..7deb02ab0 100644 --- a/targets.toml +++ b/targets.toml @@ -11,6 +11,7 @@ deploy = true [[target]] target = "x86_64-unknown-linux-gnu" +platforms = ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"] os = "ubuntu-latest" cpp = true dylib = true @@ -47,6 +48,7 @@ runners = "native qemu-user qemu-system" [[target]] target = "aarch64-unknown-linux-gnu" +platforms = ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"] os = "ubuntu-latest" cpp = true dylib = true @@ -532,8 +534,8 @@ special = true [[target]] target = "aarch64-unknown-linux-gnu" -platforms = ["x86_64-unknown-linux-gnu"] sub = "centos" +platforms = ["x86_64-unknown-linux-gnu"] os = "ubuntu-latest" cpp = true dylib = true diff --git a/xtask/src/ci/target_matrix.rs b/xtask/src/ci/target_matrix.rs index a9c93e31a..50157ee5f 100644 --- a/xtask/src/ci/target_matrix.rs +++ b/xtask/src/ci/target_matrix.rs @@ -62,6 +62,7 @@ impl TargetMatrix { runners: vec![], none: false, has_image: true, + platform: vec![], verbose: false, tests: vec!["all".to_owned()], }, @@ -285,6 +286,8 @@ struct TargetMatrixArgs { none: bool, #[clap(long)] has_image: bool, + #[clap(long, num_args = 0..)] + platform: Vec, #[clap(long, short)] verbose: bool, #[clap(long, value_parser = PossibleValuesParser::new(&[ @@ -314,6 +317,7 @@ impl Default for TargetMatrixArgs { runners: Vec::new(), none: false, has_image: false, + platform: Vec::new(), verbose: false, tests: vec!["all".to_owned()], } @@ -367,6 +371,22 @@ impl TargetMatrixArgs { .any(|runner| m.runners.as_deref().unwrap_or_default().contains(runner)) }); } + if !self.platform.is_empty() { + matrix.retain(|t| { + t.platforms() + .iter() + .any(|platform| self.platform.contains(platform)) + }); + matrix.iter_mut().for_each(|t| { + t.platforms = Some( + t.platforms() + .iter() + .filter(|&p| self.platform.contains(p)) + .cloned() + .collect(), + ) + }); + } } fn tests(&self) -> Result { diff --git a/xtask/src/codegen.rs b/xtask/src/codegen.rs index eafaed4e7..5d92a0d90 100644 --- a/xtask/src/codegen.rs +++ b/xtask/src/codegen.rs @@ -1,5 +1,4 @@ use clap::Args; -use cross::docker::ImagePlatform; use eyre::Context; use std::fmt::Write; diff --git a/xtask/src/util.rs b/xtask/src/util.rs index dea687a34..5d35fa261 100644 --- a/xtask/src/util.rs +++ b/xtask/src/util.rs @@ -4,8 +4,8 @@ use std::io::Write; use std::path::{Path, PathBuf}; use std::process::Command; -use cross::{docker::ImagePlatform, shell::MessageInfo}; use cross::{docker, CommandExt, ToUtf8}; +use cross::{docker::ImagePlatform, shell::MessageInfo}; use once_cell::sync::{Lazy, OnceCell}; use serde::Deserialize; @@ -387,6 +387,7 @@ mod tests { ImageTarget { name: "x86_64-unknown-linux-gnu".to_owned(), sub: None, + platform: None, }, "x86_64-unknown-linux-gnu".parse().unwrap() ); @@ -394,6 +395,7 @@ mod tests { ImageTarget { name: "x86_64-unknown-linux-gnu".to_owned(), sub: Some("centos".to_owned()), + platform: None, }, "x86_64-unknown-linux-gnu.centos".parse().unwrap() ); @@ -401,6 +403,7 @@ mod tests { ImageTarget { name: "thumbv8m.main-none-eabihf".to_owned(), sub: None, + platform: None, }, "thumbv8m.main-none-eabihf".parse().unwrap() ); @@ -408,6 +411,7 @@ mod tests { ImageTarget { name: "thumbv8m.main-unknown-linux-gnueabihf".to_owned(), sub: Some("alpine".to_owned()), + platform: None, }, "thumbv8m.main-unknown-linux-gnueabihf.alpine" .parse() From b53b433d9b17a07740de6613d36b9ccadeef5a4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Gardstr=C3=B6m?= Date: Mon, 3 Mar 2025 21:40:35 +0100 Subject: [PATCH 3/4] codegen bump --- src/docker/provided_images.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/docker/provided_images.rs b/src/docker/provided_images.rs index ba55026a9..1a514c87e 100644 --- a/src/docker/provided_images.rs +++ b/src/docker/provided_images.rs @@ -5,7 +5,7 @@ use super::{ImagePlatform, ProvidedImage}; pub static PROVIDED_IMAGES: &[ProvidedImage] = &[ ProvidedImage { name: "x86_64-unknown-linux-gnu", - platforms: &[ImagePlatform::X86_64_UNKNOWN_LINUX_GNU], + platforms: &[ImagePlatform::X86_64_UNKNOWN_LINUX_GNU, ImagePlatform::AARCH64_UNKNOWN_LINUX_GNU], sub: None }, ProvidedImage { @@ -20,7 +20,7 @@ pub static PROVIDED_IMAGES: &[ProvidedImage] = &[ }, ProvidedImage { name: "aarch64-unknown-linux-gnu", - platforms: &[ImagePlatform::X86_64_UNKNOWN_LINUX_GNU], + platforms: &[ImagePlatform::X86_64_UNKNOWN_LINUX_GNU, ImagePlatform::AARCH64_UNKNOWN_LINUX_GNU], sub: None }, ProvidedImage { From a528c30ee17e76c8b1d73fc50c769692de586348 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Gardstr=C3=B6m?= Date: Mon, 3 Mar 2025 23:06:34 +0100 Subject: [PATCH 4/4] seems we actually want qemu sometimes :) --- docker/qemu.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docker/qemu.sh b/docker/qemu.sh index d364983ac..01a0b4e2e 100755 --- a/docker/qemu.sh +++ b/docker/qemu.sh @@ -140,12 +140,6 @@ main() { local arch="${1}" \ softmmu="${2:-}" - # if arch is what we're currently running, we can just ignore qemu. - if [ "$(uname -m)" = "${arch}" ]; then - echo "Already running on ${arch}, skipping QEMU installation." - exit 0 - fi - install_packages \ autoconf \ automake \