Merge branch 'main' into etiotto.remove_masks

Author: etiotto

.github/workflows/bandit-check.yml

@@ -32,7 +32,7 @@ jobs:
           bandit -c bandit.yaml -r ./benchmarks/ ./scripts/ ./third_party/intel/ --exit-zero -f html -o bandit_report.html
 
       - name: Upload report to artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: Bandit Report
           path: bandit_report.html

.github/workflows/build-test-reusable.yml

@@ -127,6 +127,10 @@ jobs:
             DEBUG=1
             python -m build --wheel --no-isolation && pip install dist/*.whl
 
+      - name: Build PTI
+        run: |
+          ./scripts/install-pti.sh --build-level-zero
+
       - name: Set test-triton command line
         id: test-triton
         run: |
@@ -170,13 +174,19 @@ jobs:
           dest: ${{ steps.pip-cache.outputs.dest }}
 
       - name: Upload Triton wheels
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: triton-${{ inputs.python_version }}-${{ inputs.runner_label || inputs.driver_version }}
           path: dist/*.whl
 
-      - name: Upload test reports
+      - name: Upload PTI wheels
         uses: actions/upload-artifact@v4
+        with:
+          name: pti-${{ inputs.python_version }}-${{ inputs.runner_label || inputs.driver_version }}
+          path: .scripts_cache/pti/dist/*.whl
+
+      - name: Upload test reports
+        uses: actions/upload-artifact@v5
         with:
           name: test-reports-build-${{ inputs.python_version }}-${{ inputs.runner_label || inputs.driver_version }}
           include-hidden-files: true
@@ -238,15 +248,27 @@ jobs:
           mode: ${{ inputs.pytorch_mode }}
 
       - name: Download Triton wheels
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: triton-${{ inputs.python_version }}-${{ inputs.runner_label || inputs.driver_version }}
 
+      - name: Download PTI wheels
+        uses: actions/download-artifact@v5
+        with:
+          name: pti-${{ inputs.python_version }}-${{ inputs.runner_label || inputs.driver_version }}
+
       - name: Install Triton
         run: |
           pip install triton-*.whl
           python -c 'import triton; print(triton.__version__)'
 
+      - name: Install PTI
+        run: |
+          pip install intel_pti-*.whl
+          PTI_LIBS_DIR=$(python ./scripts/pti_lib.py)
+          ls $PTI_LIBS_DIR
+          echo "PTI_LIBS_DIR=$PTI_LIBS_DIR" | tee -a $GITHUB_ENV
+
       - name: Report environment details
         run: |
           if [ "${{ matrix.suite }}" == "minicore" ]; then
@@ -283,19 +305,7 @@ jobs:
       - name: Build PTI && Run Proton tests
         if: matrix.suite == 'rest'
         run: |
-          PTI_COMMIT_ID="$(<.github/pins/pti.txt)"
-          git clone https://github.com/intel/pti-gpu.git
-          cd pti-gpu
-          git checkout $PTI_COMMIT_ID
-          cd sdk
-          cmake --preset linux-icpx-release
-          BUILD_TESTING=1 PTI_BUILD_SAMPLES=1 cmake --build --preset linux-icpx-release
-
-          PTI_LIBS_DIR="$(pwd)/build-linux-icpx-release/lib/"
-          cd ../..
-
-          export LD_LIBRARY_PATH=$PTI_LIBS_DIR:$LD_LIBRARY_PATH
-          export TRITON_XPUPTI_LIB_PATH=$PTI_LIBS_DIR
+          export LD_LIBRARY_PATH=${{ env.PTI_LIBS_DIR }}:$LD_LIBRARY_PATH
           cd third_party/proton/test
           pytest test_api.py test_cmd.py test_lib.py test_profile.py test_viewer.py --device xpu -s -v
           cd ..
@@ -382,7 +392,7 @@ jobs:
           dest: ${{ steps.pip-cache.outputs.dest }}
 
       - name: Upload test reports
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: test-reports-${{ matrix.suite }}-${{ inputs.python_version }}-${{ inputs.runner_label || inputs.driver_version }}
           include-hidden-files: true
@@ -405,7 +415,7 @@ jobs:
           key: pip-${{ inputs.python_version }}-${{ hashFiles('pyproject.toml', 'setup.py') }}-${{ env.PIP_CACHE_NUMBER }}
 
       - name: Download test reports
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           pattern: test-reports-*-${{ inputs.python_version }}-${{ inputs.runner_label || inputs.driver_version }}
           path: reports
@@ -431,20 +441,20 @@ jobs:
           python scripts/pass_rate.py --reports reports --suite tutorials --json > pass_rate_tutorials.json
 
       - name: Upload pass rate report
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: pass_rate-${{ inputs.python_version }}-${{ inputs.runner_label || inputs.driver_version }}
           path: pass_rate*.json
 
       - name: Upload tutorials test report
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: test-reports-tutorials-${{ inputs.python_version }}-${{ inputs.runner_label || inputs.driver_version }}
           include-hidden-files: true
           path: reports/tutorials.xml
 
       - name: Upload tutorials performance report
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: tutorials-${{ inputs.python_version }}-${{ inputs.runner_label || inputs.driver_version }}
           if-no-files-found: warn

.github/workflows/build-test-windows.yml

@@ -202,13 +202,13 @@ jobs:
       - name: Upload pass rate report
         # upload reports only for the default branch
         if: github.ref_name == 'main'
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: pass_rate
           path: ${{ env.NEW_WORKSPACE }}\pass_rate*.json
 
       - name: Upload tutorial performance
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: tutorials-report
           if-no-files-found: warn

.github/workflows/create_release.yml

@@ -64,7 +64,7 @@ jobs:
           files: dist/${{env.RELEASE_FILE}}
       - name: Upload source distribution to GHA artifacts for release tags
         if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && contains(github.ref, 'rc') }}
-        uses: actions/upload-artifact@v4.4.0
+        uses: actions/upload-artifact@v5
         with:
           name: ${{ env.RELEASE_FILE }}
           path: dist/${{ env.RELEASE_FILE }}

.github/workflows/e2e-accuracy.yml

@@ -125,6 +125,8 @@ jobs:
         dtype: ${{ fromJson(needs.setup.outputs.dtype) }}
       fail-fast: false
     uses: ./.github/workflows/e2e-reusable.yml
+    secrets:
+      HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
     with:
       pytorch_ref: ${{ inputs.pytorch_ref || '' }}
       suite: ${{ matrix.suite }}
@@ -152,7 +154,7 @@ jobs:
           python-version: "3.10"
 
       - name: Download all artifacts
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           path: separate-reports
           merge-multiple: true
@@ -168,7 +170,7 @@ jobs:
              --output-dir aggregated-results
 
       - name: Upload aggregated results
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         if: ${{ !cancelled() }}
         with:
           name: aggregated-results-${{ github.run_id }}

.github/workflows/e2e-reusable.yml

@@ -2,6 +2,9 @@ name: E2E reusable workflow
 
 on:
   workflow_call:
+    secrets:
+      HUGGING_FACE_HUB_TOKEN:
+        required: true
     inputs:
       pytorch_ref:
         description: PyTorch ref, keep empty for default
@@ -137,7 +140,7 @@ jobs:
 
       - name: Install python test dependencies
         run: |
-          pip install pyyaml pandas scipy 'numpy==1.26.4' psutil pyre_extensions torchrec
+          pip install pyyaml pandas scipy psutil
 
       - name: Install transformers package
         if: ${{ inputs.suite == 'huggingface' }}
@@ -186,11 +189,19 @@ jobs:
         if: ${{ inputs.suite == 'torchbench' }}
         run: |
           cd benchmark
+          sed -i 's/^ *pynvml.*//' requirements.txt
+          pip install -r requirements.txt
+          # for dlrm
+          pip install pyre-extensions
+          curl -fsSL https://raw.githubusercontent.com/facebookresearch/dlrm/refs/heads/torchrec-dlrm/requirements.txt |xargs pip install
+          # for soft_actor_critic, temp fix
+          pip install git+https://github.com/nocoding03/gym@fix-np
+
           if [[ "${{ inputs.only_one_model }}" ]]; then
             python install.py "${{ inputs.only_one_model }}"
           else
             # install all models
-            python install.py
+            python install.py --continue_on_fail
           fi
           pip install -e .
 
@@ -200,6 +211,9 @@ jobs:
         run: |
           cd pytorch
 
+          # Some models are still not compatible with numpy>2.0
+          pip install 'numpy==1.26.4'
+
           export WORKSPACE=$GITHUB_WORKSPACE
 
           if [[ "${{ inputs.TORCH_COMPILE_DEBUG }}" = "1" ]] ; then
@@ -277,7 +291,7 @@ jobs:
           fi
 
       - name: Upload test logs
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: logs-${{ inputs.suite }}-${{ inputs.dtype }}-${{ inputs.mode }}-${{ inputs.test_mode }}
           path: inductor_log

.github/workflows/e2e-windows.yml

@@ -339,7 +339,7 @@ jobs:
           EOF
 
       - name: Upload test logs
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: logs
           path: ${{ env.NEW_WORKSPACE }}\inductor_log

.github/workflows/inductor-tests-reusable.yml

@@ -102,7 +102,7 @@ jobs:
 
       - name: Upload test logs
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: logs-${{ env.PYTHON_VERSION }}
           path: pytorch/test/test-reports

.github/workflows/inductor-tests-windows.yml

@@ -149,7 +149,7 @@ jobs:
 
       - name: Upload test logs
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: logs-${{ env.PYTHON_VERSION }}
           path: ${{ env.NEW_WORKSPACE }}\pytorch\test\test-reports

.github/workflows/llvm-build.yml

@@ -258,7 +258,7 @@ jobs:
         docker rm "${CONTAINER_ID}"
 
     - name: Upload Build Artifacts
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v5
       with:
         name: llvm-${{ matrix.config.target-os }}-${{ matrix.config.arch }}
         path: |