Allow using glob aliases for custom try jobs

Author: Kobzol

src/ci/citool/Cargo.lock

@@ -107,6 +107,7 @@ dependencies = [
  "build_helper",
  "clap",
  "csv",
+ "glob-match",
  "insta",
  "serde",
  "serde_json",
@@ -308,6 +309,12 @@ dependencies = [
  "wasi",
 ]
 
+[[package]]
+name = "glob-match"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9985c9503b412198aa4197559e9a318524ebc4519c229bfa05a535828c950b9d"
+
 [[package]]
 name = "hashbrown"
 version = "0.15.2"

src/ci/citool/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 anyhow = "1"
 clap = { version = "4.5", features = ["derive"] }
 csv = "1"
+glob-match = "0.2"
 serde = { version = "1", features = ["derive"] }
 serde_yaml = "0.9"
 serde_json = "1"

src/ci/citool/src/jobs.rs

@@ -1,7 +1,7 @@
-use crate::{GitHubContext, utils};
-use serde_yaml::Value;
+#[cfg(test)]
+mod tests;
+
 use std::collections::BTreeMap;
-use std::path::Path;
 
 use serde_yaml::Value;
 
@@ -65,13 +65,19 @@ pub struct JobDatabase {
 }
 
 impl JobDatabase {
-    fn find_auto_job_by_name(&self, name: &str) -> Option<Job> {
-        self.auto_jobs.iter().find(|j| j.name == name).cloned()
+    /// Find `auto` jobs that correspond to the passed `pattern`.
+    /// Patterns are matched using the glob syntax.
+    /// For example `dist-*` matches all jobs starting with `dist-`.
+    fn find_auto_jobs_by_pattern(&self, pattern: &str) -> Vec<Job> {
+        self.auto_jobs
+            .iter()
+            .filter(|j| glob_match::glob_match(pattern, &j.name))
+            .cloned()
+            .collect()
     }
 }
 
-pub fn load_job_db(path: &Path) -> anyhow::Result<JobDatabase> {
-    let db = utils::read_to_string(path)?;
+pub fn load_job_db(db: &str) -> anyhow::Result<JobDatabase> {
     let mut db: Value = serde_yaml::from_str(&db)?;
 
     // We need to expand merge keys (<<), because serde_yaml can't deal with them
@@ -114,7 +120,7 @@ pub enum RunType {
     /// Workflows that run after a push to a PR branch
     PullRequest,
     /// Try run started with @bors try
-    TryJob { custom_jobs: Option<Vec<String>> },
+    TryJob { job_patterns: Option<Vec<String>> },
     /// Merge attempt workflow
     AutoJob,
 }
@@ -130,28 +136,29 @@ fn calculate_jobs(
 ) -> anyhow::Result<Vec<GithubActionsJob>> {
     let (jobs, prefix, base_env) = match run_type {
         RunType::PullRequest => (db.pr_jobs.clone(), "PR", &db.envs.pr_env),
-        RunType::TryJob { custom_jobs } => {
-            let jobs = if let Some(custom_jobs) = custom_jobs {
-                if custom_jobs.len() > MAX_TRY_JOBS_COUNT {
-                    return Err(anyhow::anyhow!(
-                        "It is only possible to schedule up to {MAX_TRY_JOBS_COUNT} custom jobs, received {} custom jobs",
-                        custom_jobs.len()
-                    ));
-                }
-
-                let mut jobs = vec![];
-                let mut unknown_jobs = vec![];
-                for custom_job in custom_jobs {
-                    if let Some(job) = db.find_auto_job_by_name(custom_job) {
-                        jobs.push(job);
+        RunType::TryJob { job_patterns } => {
+            let jobs = if let Some(patterns) = job_patterns {
+                let mut jobs: Vec<Job> = vec![];
+                let mut unknown_patterns = vec![];
+                for pattern in patterns {
+                    let matched_jobs = db.find_auto_jobs_by_pattern(pattern);
+                    if matched_jobs.is_empty() {
+                        unknown_patterns.push(pattern.clone());
                     } else {
-                        unknown_jobs.push(custom_job.clone());
+                        jobs.extend(matched_jobs);
                     }
                 }
-                if !unknown_jobs.is_empty() {
+                if !unknown_patterns.is_empty() {
+                    return Err(anyhow::anyhow!(
+                        "Patterns `{}` did not match any auto jobs",
+                        unknown_patterns.join(", ")
+                    ));
+                }
+                if jobs.len() > MAX_TRY_JOBS_COUNT {
                     return Err(anyhow::anyhow!(
-                        "Custom job(s) `{}` not found in auto jobs",
-                        unknown_jobs.join(", ")
+                        "It is only possible to schedule up to {MAX_TRY_JOBS_COUNT} custom jobs, received {} custom jobs expanded from {} pattern(s)",
+                        jobs.len(),
+                        patterns.len()
                     ));
                 }
                 jobs

src/ci/citool/src/jobs/tests.rs

@@ -0,0 +1,64 @@
+use crate::jobs::{JobDatabase, load_job_db};
+
+#[test]
+fn lookup_job_pattern() {
+    let db = load_job_db(
+        r#"
+envs:
+  pr:
+  try:
+  auto:
+
+pr:
+try:
+auto:
+    - name: dist-a
+      os: ubuntu
+      env: {}
+    - name: dist-a-alt
+      os: ubuntu
+      env: {}
+    - name: dist-b
+      os: ubuntu
+      env: {}
+    - name: dist-b-alt
+      os: ubuntu
+      env: {}
+    - name: test-a
+      os: ubuntu
+      env: {}
+    - name: test-a-alt
+      os: ubuntu
+      env: {}
+    - name: test-i686
+      os: ubuntu
+      env: {}
+    - name: dist-i686
+      os: ubuntu
+      env: {}
+    - name: test-msvc-i686-1
+      os: ubuntu
+      env: {}
+    - name: test-msvc-i686-2
+      os: ubuntu
+      env: {}
+"#,
+    )
+    .unwrap();
+    check_pattern(&db, "dist-*", &["dist-a", "dist-a-alt", "dist-b", "dist-b-alt", "dist-i686"]);
+    check_pattern(&db, "*-alt", &["dist-a-alt", "dist-b-alt", "test-a-alt"]);
+    check_pattern(&db, "dist*-alt", &["dist-a-alt", "dist-b-alt"]);
+    check_pattern(
+        &db,
+        "*i686*",
+        &["test-i686", "dist-i686", "test-msvc-i686-1", "test-msvc-i686-2"],
+    );
+}
+
+#[track_caller]
+fn check_pattern(db: &JobDatabase, pattern: &str, expected: &[&str]) {
+    let jobs =
+        db.find_auto_jobs_by_pattern(pattern).into_iter().map(|j| j.name).collect::<Vec<_>>();
+
+    assert_eq!(jobs, expected);
+}

src/ci/citool/src/main.rs

@@ -35,21 +35,21 @@ impl GitHubContext {
     fn get_run_type(&self) -> Option<RunType> {
         match (self.event_name.as_str(), self.branch_ref.as_str()) {
             ("pull_request", _) => Some(RunType::PullRequest),
-            ("push", "refs/heads/try-perf") => Some(RunType::TryJob { custom_jobs: None }),
+            ("push", "refs/heads/try-perf") => Some(RunType::TryJob { job_patterns: None }),
             ("push", "refs/heads/try" | "refs/heads/automation/bors/try") => {
-                let custom_jobs = self.get_custom_jobs();
-                let custom_jobs = if !custom_jobs.is_empty() { Some(custom_jobs) } else { None };
-                Some(RunType::TryJob { custom_jobs })
+                let patterns = self.get_try_job_patterns();
+                let patterns = if !patterns.is_empty() { Some(patterns) } else { None };
+                Some(RunType::TryJob { job_patterns: patterns })
             }
             ("push", "refs/heads/auto") => Some(RunType::AutoJob),
             _ => None,
         }
     }
 
-    /// Tries to parse names of specific CI jobs that should be executed in the form of
-    /// try-job: <job-name>
+    /// Tries to parse patterns of CI jobs that should be executed in the form of
+    /// try-job: <job-pattern>
     /// from the commit message of the passed GitHub context.
-    fn get_custom_jobs(&self) -> Vec<String> {
+    fn get_try_job_patterns(&self) -> Vec<String> {
         if let Some(ref msg) = self.commit_message {
             msg.lines()
                 .filter_map(|line| line.trim().strip_prefix("try-job: "))
@@ -180,7 +180,10 @@ pub enum JobType {
 fn main() -> anyhow::Result<()> {
     let args = Args::parse();
     let default_jobs_file = Path::new(JOBS_YML_PATH);
-    let load_db = |jobs_path| jobs::load_job_db(jobs_path).context("Cannot load jobs.yml");
+    let load_db = |jobs_path| {
+        let db = utils::read_to_string(jobs_path)?;
+        Ok::<_, anyhow::Error>(jobs::load_job_db(&db).context("Cannot load jobs.yml")?)
+    };
 
     match args {
         Args::CalculateJobMatrix { jobs_file } => {