Merge #1091

1091: improve weekly checks and push the resulting images r=Alexhuszagh a=Emilgardis

since the maximum timeout for workflows is not enough, we need to use the same strategy as our normal ci flow

also adds the ability to filter targets to run

<img width="366" alt="image" src="https://user-images.githubusercontent.com/1502855/197333979-2ad4616e-50ff-4b9a-aea1-dfdef518fe3c.png">


this pr also pushes the results so that the cache is renewed, making the images not stagnate

Co-authored-by: Emil Gardström <[email protected]>

Author: bors[bot]

.github/workflows/weekly.yml

@@ -2,6 +2,10 @@ on:
   schedule:
     - cron:  '0 0 * * 5'
   workflow_dispatch:
+    inputs:
+      targets:
+        required: false
+        description: 'check these space or comma separated targets, supports wildcard *'
 
 name: Check
 
@@ -10,21 +14,116 @@ env:
   CARGO_HTTP_CHECK_REVOKE: false
 
 jobs:
+  generate-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.generate-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-rust
+
+      - name: Generate matrix
+        id: generate-matrix
+        run: |
+          cargo xtask ci-job target-matrix --author "[gha]" --message "check" --weekly
+        env:
+          TARGETS: ${{ inputs.targets }}
   weekly:
-    name: Check All Targets - No Cache
+    name: Check target - No Cache (${{ matrix.pretty }},${{ matrix.os }})
     timeout-minutes: 1440
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    needs: generate-matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/setup-rust
       - name: Set up Docker Buildx
+        if: runner.os == 'Linux'
         uses: docker/setup-buildx-action@v2
       - name: Build xtask
         run: cargo build -p xtask
+
+      - name: Prepare Meta
+        id: prepare-meta
+        timeout-minutes: 60
+        run: cargo xtask ci-job prepare-meta "${TARGET}${SUB:+.$SUB}"
+        env:
+          TARGET: ${{ matrix.target }}
+          SUB: ${{ matrix.sub }}
+        shell: bash
+      - name: Docker Meta
+        if: steps.prepare-meta.outputs.has-image
+        id: docker-meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            name=${{ steps.prepare-meta.outputs.image }}
+          labels: |
+            ${{ fromJSON(steps.prepare-meta.outputs.labels) }}
+
       - name: Build Docker image
         id: build-docker-image
-        run: cargo xtask build-docker-image -v --no-cache --no-output --from-ci --no-fastfail --tag weekly
-        timeout-minutes: 1440
+        if: steps.prepare-meta.outputs.has-image
+        timeout-minutes: 120
+        run: cargo xtask build-docker-image --no-cache -v "${TARGET}${SUB:+.$SUB}"
+        env:
+          TARGET: ${{ matrix.target }}
+          SUB: ${{ matrix.sub }}
+        shell: bash
+      - name: Set Docker image for test
+        if: steps.prepare-meta.outputs.has-image
+        run: |
+          TARGET_VAR="cross_target_${TARGET//-/_}_image"
+          echo "${TARGET_VAR^^}=${IMAGE}" | tee -a "${GITHUB_ENV}"
+        env:
+          TARGET: ${{ matrix.target }}
+          IMAGE: ${{ steps.build-docker-image.outputs.image }}
+        shell: bash
+      - name: Test Image
+        if: steps.prepare-meta.outputs.has-image && steps.prepare-meta.outputs.test-variant == 'default'
+        run: ./ci/test.sh
+        env:
+          TARGET: ${{ matrix.target }}
+          CPP: ${{ matrix.cpp }}
+          DYLIB: ${{ matrix.dylib }}
+          STD: ${{ matrix.std }}
+          BUILD_STD: ${{ matrix.build-std }}
+          RUN: ${{ matrix.run }}
+          RUNNERS: ${{ matrix.runners }}
+        shell: bash
+
+      - name: Test Zig Image
+        if: steps.prepare-meta.outputs.has-image && steps.prepare-meta.outputs.test-variant == 'zig'
+        run: ./ci/test-zig-image.sh
+        shell: bash
+
+      - name: Test Cross Image
+        if: steps.prepare-meta.outputs.has-image && steps.prepare-meta.outputs.test-variant == 'cross'
+        run: ./ci/test-cross-image.sh
+        env:
+          TARGET: 'aarch64-unknown-linux-gnu'
+          IMAGE: 'ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main'
+        shell: bash
+
+      - name: Login to GitHub Container Registry
+        if: steps.prepare-meta.outputs.has-image && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push
+        if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
+        run: cargo xtask build-docker-image -v --push "${TARGET}${SUB:+.$SUB}"
+        env:
+          TARGET: ${{ matrix.target }}
+          SUB: ${{ matrix.sub }}
+          LABELS: ${{ steps.docker-meta.outputs.labels }}
+        shell: bash
   wiki:
     name: Ensure wiki is valid
     runs-on: ubuntu-latest

xtask/src/ci.rs

@@ -31,6 +31,9 @@ pub enum CiJob {
         message: String,
         #[clap(long, env = "COMMIT_AUTHOR")]
         author: String,
+        // check is being run as part of a weekly check
+        #[clap(long)]
+        weekly: bool,
     },
 }
 
@@ -114,8 +117,12 @@ pub fn ci(args: CiJob, metadata: CargoMetadata) -> cross::Result<()> {
                 }
             }
         }
-        CiJob::TargetMatrix { message, author } => {
-            target_matrix::run(message, author)?;
+        CiJob::TargetMatrix {
+            message,
+            author,
+            weekly,
+        } => {
+            target_matrix::run(message, author, weekly)?;
         }
     }
     Ok(())

xtask/src/ci/target_matrix.rs

@@ -7,10 +7,28 @@ use serde::{Deserialize, Serialize};
 
 use crate::util::{get_matrix, gha_output, gha_print, CiTarget, ImageTarget};
 
-pub(crate) fn run(message: String, author: String) -> Result<(), color_eyre::Report> {
+pub(crate) fn run(message: String, author: String, weekly: bool) -> Result<(), color_eyre::Report> {
     let mut matrix: Vec<CiTarget> = get_matrix().clone();
     let (prs, mut app) = if author == "bors[bot]" {
         process_bors_message(&message)?
+    } else if weekly {
+        let app = TargetMatrixArgs {
+            target: std::env::var("TARGETS")
+                .unwrap_or_default()
+                .split(' ')
+                .flat_map(|s| s.split(','))
+                .filter(|s| !s.is_empty())
+                .map(|s| s.to_owned())
+                .collect(),
+            std: None,
+            cpp: None,
+            dylib: None,
+            run: None,
+            runners: vec![],
+            none: false,
+            has_image: true,
+        };
+        (vec![], app)
     } else {
         (vec![], TargetMatrixArgs::default())
     };
@@ -146,6 +164,8 @@ struct TargetMatrixArgs {
     runners: Vec<String>,
     #[clap(long)]
     none: bool,
+    #[clap(long)]
+    has_image: bool,
 }
 
 impl TargetMatrixArgs {
@@ -158,6 +178,9 @@ impl TargetMatrixArgs {
             std::mem::take(matrix);
             return;
         }
+        if self.has_image {
+            matrix.retain(|t| t.to_image_target().has_ci_image());
+        }
         if !self.target.is_empty() {
             matrix.retain(|m| {
                 let matrix_target = m.to_image_target();