Workflow to run E2E tests on Windows (#4518)

The same parameters as E2E on Linux, but without parallel jobs (yet).

Fixes #3591.

---------

Signed-off-by: Pavel Chekin <[email protected]>

Author: pbchekin

.github/workflows/e2e-reusable.yml

@@ -215,16 +215,16 @@ jobs:
         env:
           HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
         run: |
+          cd pytorch
+
           export WORKSPACE=$GITHUB_WORKSPACE
 
-          if [[ "${{ inputs.TORCH_COMPILE_DEBUG }}" == "1" ]] ; then
+          if [[ "${{ inputs.TORCH_COMPILE_DEBUG }}" = "1" ]] ; then
             export TORCH_COMPILE_DEBUG="1"
             # torch will save debug logs to $TORCH_COMPILE_DEBUG_DIR/torch_compile_debug
             export TORCH_COMPILE_DEBUG_DIR=$GITHUB_WORKSPACE
           fi
 
-          cd pytorch
-
           # if "only_one_model" is set, then test this model
           # if "models" == "subset", then test the models from .github/models/{accuracy,performance}/{suite}.txt
           # otherwise test all models.

.github/workflows/e2e-windows.yml

@@ -224,6 +224,101 @@ jobs:
           extra-cache-key: ${{ env.PYTORCH_VERSION }}
           workspace: /c/gh${{ github.run_id }}
 
+      - name: Clone pytorch benchmark
+        if: inputs.suite == 'all' || inputs.suite == 'torchbench'
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ env.BENCHMARK_REPO }}
+          ref: ${{ env.BENCHMARK_COMMIT_ID }}
+          submodules: recursive
+          path: benchmark
+
+      - name: Install pytorch benchmark
+        if: inputs.suite == 'all' || inputs.suite == 'torchbench'
+        run: |
+          .venv\Scripts\activate.ps1
+          Invoke-BatchFile "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
+          bash -c '
+          cd benchmark
+          if [[ "${{ inputs.only_one_model }}" ]]; then
+            python install.py "${{ inputs.only_one_model }}"
+          else
+            # install all models
+            python install.py
+          fi
+          pip install -e .
+          '
+
+      - name: Run e2e tests
+        env:
+          HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
+        run: |
+          .venv\Scripts\activate.ps1
+          Invoke-BatchFile "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
+
+          bash -c '
+          cd /c/pytorch
+
+          export WORKSPACE="/c/gh${{ github.run_id }}"
+
+          if [[ "${{ inputs.TORCH_COMPILE_DEBUG }}" = "1" ]] ; then
+            export TORCH_COMPILE_DEBUG="1"
+            # torch will save debug logs to $TORCH_COMPILE_DEBUG_DIR/torch_compile_debug
+            export TORCH_COMPILE_DEBUG_DIR=$WORKSPACE
+          fi
+
+          if [[ "${{ inputs.suite }}" = "all" ]]; then
+            suites=("huggingface" "timm_models" "torchbench")
+          else
+            suites=("${{ inputs.suite }}")
+          fi
+
+          if [[ "${{ inputs.mode }}" = "all" ]]; then
+            modes=("inference" "inference-with-freezing" "training")
+          else
+            modes=("${{ inputs.mode }}")
+          fi
+
+          if [[ "${{ inputs.dtype }}" = "all" ]]; then
+            dtypes=("amp_bf16" "amp_fp16" "bfloat16" "float16" "float32")
+          else
+            dtypes=("${{ inputs.dtype }}")
+          fi
+
+          # if "only_one_model" is set, then test this model
+          # if "models" == "subset", then test the models from .github/models/{accuracy,performance}/{suite}.txt
+          # otherwise test all models.
+
+          for suite in ${suites[@]}; do
+            for mode in ${modes[@]}; do
+              for dtype in ${dtypes[@]}; do
+
+                if [[ "${{ inputs.only_one_model }}" ]]; then
+                  bash -e $WORKSPACE/scripts/inductor_xpu_test.sh $suite $dtype $mode ${{ inputs.test_mode }} xpu 0 static 1 0 ${{ inputs.only_one_model }}
+                elif [[ "${{ inputs.models }}" == "subset" ]]; then
+                  models_subset_file="$WORKSPACE/.github/models/${{ inputs.test_mode }}/$suite.txt"
+                  while read model; do
+                    bash -e $WORKSPACE/scripts/inductor_xpu_test.sh $suite $dtype $mode ${{ inputs.test_mode }} xpu 0 static 1 0 $model
+                  done < $models_subset_file
+                  if [[ "${{ inputs.check_all_subset_models }}" == true ]]; then
+                    python $WORKSPACE/scripts/check_inductor_report.py --models-file="$models_subset_file" \
+                      --suite=$suite \
+                      --dtype=$dtype \
+                      --mode=$mode \
+                      --test_mode=${{ inputs.test_mode }} \
+                      --device=xpu \
+                      --inductor-log-dir="$WORKSPACE/inductor_log"
+                  fi
+                else
+                  bash -e $WORKSPACE/scripts/inductor_xpu_test.sh $suite $dtype $mode ${{ inputs.test_mode }} xpu 0 static 1 0
+                fi
+
+              done
+            done
+          done
+          '
+
+
       - name: Identify GPU
         run: |
           # Initializing oneAPI to enable sycl-ls, which is used in capture-hw-details.sh on Windows.