Add test_all_subset_models mode

.github/workflows/e2e-accuracy.yml

@@ -44,6 +44,10 @@ on:
           - all
           - subset
         default: all
+      test_all_subset_models:
+        description: In "subset" mode, do not fail workflow if one of models failed
+        type: boolean
+        default: false
       only_one_model:
         description: Run only this one model
         type: string
@@ -125,6 +129,7 @@ jobs:
       test_mode: accuracy
       dtype: ${{ matrix.dtype }}
       models: ${{ inputs.models }}
+      test_all_subset_models: ${{ inputs.test_all_subset_models || false }}
       only_one_model: ${{ inputs.only_one_model }}
       runner_label: ${{ inputs.runner_label }}
       TORCH_COMPILE_DEBUG: ${{ inputs.TORCH_COMPILE_DEBUG }}

.github/workflows/e2e-performance.yml

@@ -44,6 +44,10 @@ on:
           - all
           - subset
         default: subset
+      test_all_subset_models:
+        description: In "subset" mode, do not fail workflow if one of models failed
+        type: boolean
+        default: false
       only_one_model:
         description: Run only this one model
         type: string
@@ -136,6 +140,7 @@ jobs:
       test_mode: performance
       dtype: ${{ matrix.dtype }}
       models: ${{ inputs.models }}
+      test_all_subset_models: ${{ inputs.test_all_subset_models || false }}
       only_one_model: ${{ inputs.only_one_model }}
       runner_label: ${{ inputs.runner_label }}
       TORCH_COMPILE_DEBUG: ${{ inputs.TORCH_COMPILE_DEBUG }}

.github/workflows/e2e-reusable.yml

@@ -27,6 +27,10 @@ on:
         description: Run all models or a subset
         type: string
         default: all
+      test_all_subset_models:
+        description: In "subset" mode, do not fail workflow if one of models failed
+        type: boolean
+        default: false
       only_one_model:
         description: Run only this one model
         type: string
@@ -226,6 +230,7 @@ jobs:
           elif [[ "${{ inputs.models }}" == "subset" ]]; then
             while read model; do
               bash -e $GITHUB_WORKSPACE/scripts/inductor_xpu_test.sh ${{ inputs.suite }} ${{ inputs.dtype }} ${{ inputs.mode }} ${{ inputs.test_mode }} xpu 0 static 1 0 $model
+              grep ,$model, $WORKSPACE/inductor_log/*/*/*.csv | grep -q ,pass, || ${{ inputs.test_all_subset_models }}
             done < $GITHUB_WORKSPACE/.github/models/performance/${{ inputs.suite }}.txt
           else
             bash -e $GITHUB_WORKSPACE/scripts/inductor_xpu_test.sh ${{ inputs.suite }} ${{ inputs.dtype }} ${{ inputs.mode }} ${{ inputs.test_mode }} xpu 0 static 1 0