parallelize weekly check

Author: Emilgardis

.github/workflows/weekly.yml

@@ -2,6 +2,10 @@ on:
   schedule:
     - cron:  '0 0 * * 5'
   workflow_dispatch:
+    inputs:
+      targets:
+        required: false
+        description: 'check these space or comma separated targets, supports wildcard *'
 
 name: Check
 
@@ -10,21 +14,54 @@ env:
   CARGO_HTTP_CHECK_REVOKE: false
 
 jobs:
+  generate-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.generate-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-rust
+
+      - name: Generate matrix
+        id: generate-matrix
+        run: |
+          cargo xtask ci-job target-matrix --author "[gha]" --message "check" --weekly
+        env:
+          TARGETS: ${{ inputs.targets }}
   weekly:
-    name: Check All Targets - No Cache
+    name: Check target - No Cache (${{ matrix.pretty }},${{ matrix.os }})
     timeout-minutes: 1440
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    needs: generate-matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/setup-rust
       - name: Set up Docker Buildx
+        if: runner.os == 'Linux'
         uses: docker/setup-buildx-action@v2
       - name: Build xtask
         run: cargo build -p xtask
+      - name: Prepare Meta
+        id: prepare-meta
+        timeout-minutes: 60
+        run: cargo xtask ci-job prepare-meta "${TARGET}${SUB:+.$SUB}"
+        env:
+          TARGET: ${{ matrix.target }}
+          SUB: ${{ matrix.sub }}
+        shell: bash
       - name: Build Docker image
         id: build-docker-image
-        run: cargo xtask build-docker-image -v --no-cache --no-output --from-ci --no-fastfail --tag weekly
-        timeout-minutes: 1440
+        if: steps.prepare-meta.outputs.has-image
+        timeout-minutes: 120
+        run: cargo xtask build-docker-image --no-cache --no-output -v "${TARGET}${SUB:+.$SUB}" --tag weekly
+        env:
+          TARGET: ${{ matrix.target }}
+          SUB: ${{ matrix.sub }}
+        shell: bash
   wiki:
     name: Ensure wiki is valid
     runs-on: ubuntu-latest

xtask/src/ci.rs

@@ -31,6 +31,9 @@ pub enum CiJob {
         message: String,
         #[clap(long, env = "COMMIT_AUTHOR")]
         author: String,
+        // check is being run as part of a weekly check
+        #[clap(long)]
+        weekly: bool,
     },
 }
 
@@ -114,8 +117,12 @@ pub fn ci(args: CiJob, metadata: CargoMetadata) -> cross::Result<()> {
                 }
             }
         }
-        CiJob::TargetMatrix { message, author } => {
-            target_matrix::run(message, author)?;
+        CiJob::TargetMatrix {
+            message,
+            author,
+            weekly,
+        } => {
+            target_matrix::run(message, author, weekly)?;
         }
     }
     Ok(())

xtask/src/ci/target_matrix.rs

@@ -7,10 +7,28 @@ use serde::{Deserialize, Serialize};
 
 use crate::util::{get_matrix, gha_output, gha_print, CiTarget, ImageTarget};
 
-pub(crate) fn run(message: String, author: String) -> Result<(), color_eyre::Report> {
+pub(crate) fn run(message: String, author: String, weekly: bool) -> Result<(), color_eyre::Report> {
     let mut matrix: Vec<CiTarget> = get_matrix().clone();
     let (prs, mut app) = if author == "bors[bot]" {
         process_bors_message(&message)?
+    } else if weekly {
+        let app = TargetMatrixArgs {
+            target: std::env::var("TARGETS")
+                .unwrap_or_default()
+                .split(' ')
+                .flat_map(|s| s.split(','))
+                .filter(|s| !s.is_empty())
+                .map(|s| s.to_owned())
+                .collect(),
+            std: None,
+            cpp: None,
+            dylib: None,
+            run: None,
+            runners: vec![],
+            none: false,
+            has_image: true,
+        };
+        (vec![], app)
     } else {
         (vec![], TargetMatrixArgs::default())
     };
@@ -146,6 +164,8 @@ struct TargetMatrixArgs {
     runners: Vec<String>,
     #[clap(long)]
     none: bool,
+    #[clap(long)]
+    has_image: bool,
 }
 
 impl TargetMatrixArgs {
@@ -158,6 +178,9 @@ impl TargetMatrixArgs {
             std::mem::take(matrix);
             return;
         }
+        if self.has_image {
+            matrix.retain(|t| t.to_image_target().has_ci_image());
+        }
         if !self.target.is_empty() {
             matrix.retain(|m| {
                 let matrix_target = m.to_image_target();