From 7045d32d35e9014b0c2e526c5e3af8657bcedce4 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 11:31:25 +0200
Subject: [PATCH 001/105] Add a test run

---
 .github/workflows/conda-build-test.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml
index c4f1f4ab95..298bafc7b5 100644
--- a/.github/workflows/conda-build-test.yml
+++ b/.github/workflows/conda-build-test.yml
@@ -4,6 +4,8 @@ on:
   workflow_dispatch:
   schedule:
     - cron: "5 2 * * *"
+  #FIXME
+  push:
 
 permissions: read-all
 

From 3cc291d24fdb86e442759e9e057db7e60c578954 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 12:46:18 +0200
Subject: [PATCH 002/105] Add a custom runner

---
 .github/workflows/build-test-reusable.yml |  17 +-
 .github/workflows/build-test.yml          |   5 +-
 .github/workflows/conda-build-test.yml    | 201 ++++++++++++----------
 scripts/runner_default.sh                 |   4 +
 4 files changed, 132 insertions(+), 95 deletions(-)
 create mode 100644 scripts/runner_default.sh

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index cc623a8100..d4d1cf01fc 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -56,6 +56,14 @@ on:
         description: Runner label for version
         type: string
         default: runner-0.0.19
+      env_manager:
+        description: Environment manager
+        type: choice
+        options:
+        - default
+        - conda
+        - nobasekit      
+        default: default
 
 permissions: read-all
 
@@ -69,17 +77,18 @@ jobs:
     runs-on: ${{ fromJson(inputs.runner_label && format('["{0}"]', inputs.runner_label) || format('["{0}", "{1}", "{2}"]', inputs.device, inputs.driver_version, inputs.runner_version)) }}
     defaults:
       run:
-        shell: bash -noprofile --norc -eo pipefail -c "source /opt/intel/oneapi/setvars.sh > /dev/null; source {0}"
+        shell: bash -noprofile --norc -eo pipefail scripts/runner_${{ inputs.env_manager }}.sh"
+        envinputs.env-c "source /opt/intel/oneapi/setvars.sh > /dev/null; source {0}"
     steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
       - name: Print inputs
         run: |
           cat <<EOF
           ${{ toJSON(inputs) }}
           EOF
 
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
       - name: Load pip cache
         id: pip-cache
         uses: ./.github/actions/load
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
index d3d9b29b56..512162c2aa 100644
--- a/.github/workflows/build-test.yml
+++ b/.github/workflows/build-test.yml
@@ -44,8 +44,9 @@ on:
     branches:
       - main
   push:
-    branches:
-      - main
+# FIXME    
+#    branches:
+#      - main
 
 permissions: read-all
 
diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml
index 298bafc7b5..dbc3f76135 100644
--- a/.github/workflows/conda-build-test.yml
+++ b/.github/workflows/conda-build-test.yml
@@ -1,114 +1,137 @@
-name: Conda build and test
+name: Build and test
+run-name: ${{ inputs.run_name }}
 
 on:
   workflow_dispatch:
-  schedule:
-    - cron: "5 2 * * *"
-  #FIXME
+    inputs:
+      runner_label:
+        description: Runner label, keep empty for default
+        type: string
+        default: ""
+      pytorch_ref:
+        description: PyTorch ref, keep empty for default
+        type: string
+        default: ""
+      pytorch_mode:
+        description: PyTorch mode, source or wheels
+        type: choice
+        options:
+          - source
+          - wheels
+        default: source
+      upload_test_reports:
+        description: Upload test reports
+        type: boolean
+        default: false
+      ignore_errors:
+        description: Ignore test errors
+        type: boolean
+        default: false
+      skip_list:
+        description: Skip list
+        type: string
+        default: ""
+      run_name:
+        description: Custom run name
+        type: string
+        default: "Build and test"
+      enable_unskip:
+        description: Ignore pytest.skip
+        type: boolean
+        default: false
+
+  pull_request:
+    branches:
+      - main
   push:
+    branches:
+      - main
 
 permissions: read-all
 
-env:
-  BACKEND: XPU
-  TRITON_DISABLE_LINE_INFO: 1
-  GH_TOKEN: ${{ github.token }}
-
 jobs:
-
-  integration-tests:
-    name: Integration tests
-    runs-on:
-      - max1100
-      - rolling
-      - runner-0.0.19
-    strategy:
-      matrix:
-        python: ${{ github.ref_name == 'main' && fromJson('["3.9", "3.10", "3.11"]') || fromJson('["3.9"]') }}
-    defaults:
-      run:
-        shell: bash -noprofile --norc -eo pipefail -c "source /opt/intel/oneapi/setvars.sh > /dev/null; source {0}"
+  pre-commit:
+    name: Pre-commit checks
+    runs-on: Linux
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Calculate env
-        run: |
-          echo $HOME/miniforge3/bin >>$GITHUB_PATH
-
-      - name: Load conda cache
-        id: conda-cache
+      - name: Load pip cache
+        id: pip-cache
         uses: ./.github/actions/load
         env:
-          CACHE_NUMBER: 5
+          # Increase this value to reset cache
+          CACHE_NUMBER: 2
         with:
-          path: $HOME/miniforge3/envs/triton
-          key: conda-py${{ matrix.python }}-${{ hashFiles('scripts/triton.yml', 'python/pyproject.toml', 'python/setup.py', '.github/pins/ipex.txt', '.github/pins/pytorch.txt') }}-${{ env.CACHE_NUMBER }}
+          path: $HOME/.cache/pip
+          key: pip-3.10-${{ hashFiles('.pre-commit-config.yaml') }}-${{ env.CACHE_NUMBER }}
 
-      - name: Update conda env
-        if: ${{ steps.conda-cache.outputs.status == 'miss' }}
-        run: |
-          conda create -n triton --override-channels -c conda-forge python=${{ matrix.python }}.*
-          conda env update -f scripts/triton.yml
-          find /opt/intel/oneapi/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
-          ln -snf /usr/include/level_zero $HOME/miniforge3/envs/triton/bin/../x86_64-conda-linux-gnu/sysroot/usr/include/level_zero
-          find /usr -name libze_\* -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
+      - name: Install Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
 
-      - name: Add conda info to log
+      - name: Run pre-commit checks
         run: |
-          conda info
-          conda list -n triton
+          set -x
+          pip install --upgrade pre-commit
 
-      - name: Install latest nightly wheels
-        uses: ./.github/actions/install-wheels
+          # TODO: ignore the first yapf failure until https://github.com/google/yapf/issues/1164 is fixed
+          python3 -m pre_commit run --all-files --verbose yapf &> /dev/null || true
+          # If first run of yapf worked and made changes reset the tree to the original state
+          git reset --hard
+
+          python3 -m pre_commit run --show-diff-on-failure --color=always --all-files --verbose
+
+      - name: Save pip cache
+        if: ${{ steps.pip-cache.outputs.status == 'miss' }}
+        uses: ./.github/actions/save
         with:
-          gh_token: ${{ secrets.GITHUB_TOKEN }}
-          install_cmd: conda run --no-capture-output -n triton pip install
-          python_version: ${{ matrix.python }}
-          wheels_pattern: 'torch-*'
+          path: ${{ steps.pip-cache.outputs.path }}
+          dest: ${{ steps.pip-cache.outputs.dest }}
 
-      - name: Build Triton
-        run: |
-          set -x
-          export DEBUG=1
-          cd python
-          conda run --no-capture-output -n triton pip install pybind11
-          conda run --no-capture-output -n triton pip install --no-build-isolation -e '.[build,tests,tutorials]'
+  prepare:
+    name: Prepare
+    runs-on: Linux
 
-      - name: Run core tests
-        env:
-          # FIXME https://github.com/intel/intel-xpu-backend-for-triton/issues/806
-          # FIXME https://github.com/intel/intel-xpu-backend-for-triton/issues/841
-          TRITON_TEST_SKIPLIST_DIR: scripts/skiplist/conda
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
+
+    steps:
+      - name: Inputs
         run: |
-          set -x
-          conda run --no-capture-output -n triton bash -v -x scripts/test-triton.sh
+          cat <<EOF
+          ${{ toJSON(inputs) }}
+          EOF
 
-      - name: Run E2E test
+      - name: Matrix
+        id: matrix
         run: |
-          cd ../pytorch || {
-            PYTORCH_COMMIT_ID=$(<.github/pins/pytorch.txt)
-            cd ..
-            git clone --single-branch -b dev/triton-test-3.0 --recurse-submodules https://github.com/Stonepia/pytorch.git
-            cd pytorch
-            git branch pin-branch $PYTORCH_COMMIT_ID
-            git switch pin-branch
-          }
-
-          TRANSFORMERS_VERSION="$(<.ci/docker/ci_commit_pins/huggingface.txt)"
-          conda run -n triton pip install pyyaml pandas scipy numpy psutil pyre_extensions torchrec transformers==$TRANSFORMERS_VERSION
-
-          # Set WORKSPACE for inductor_xpu_test.sh to make sure it creates "inductor_log" outside of pytorch cloned directory
-          export WORKSPACE=$GITHUB_WORKSPACE
-          # TODO: Find the fastest Hugging Face model
-          conda run --no-capture-output -n triton $GITHUB_WORKSPACE/scripts/inductor_xpu_test.sh huggingface float32 inference accuracy xpu 0 static 1 0 AlbertForMaskedLM
-          # The script above always returns 0, so we need an additional check to see if the accuracy test passed
-          cat $WORKSPACE/inductor_log/*/*/*.csv
-          grep AlbertForMaskedLM $WORKSPACE/inductor_log/*/*/*.csv | grep -q ,pass,
-
-      - name: Save conda cache
-        if: ${{ steps.conda-cache.outputs.status == 'miss' }}
-        uses: ./.github/actions/save
-        with:
-          path: ${{ steps.conda-cache.outputs.path }}
-          dest: ${{ steps.conda-cache.outputs.dest }}
+          if [[ -n "${{ inputs.runner_label }}" ]]; then
+            matrix='{"python": ["3.9"]}'
+          else
+            matrix='{"python": ["3.9"], "driver": ["rolling", "lts"]}'
+          fi
+          echo "matrix=$matrix" | tee -a $GITHUB_OUTPUT
+
+  integration-tests:
+    name: Integration tests matrix
+    needs: prepare
+
+    strategy:
+      matrix: ${{ fromJson(needs.prepare.outputs.matrix) }}
+
+    uses: ./.github/workflows/build-test-reusable.yml
+    with:
+      driver_version: ${{ matrix.driver }}
+      runner_label: ${{ inputs.runner_label }}
+      pytorch_ref: ${{ inputs.pytorch_ref }}
+      pytorch_mode: ${{ inputs.pytorch_mode || 'source' }}
+      python_version: ${{ matrix.python }}
+      upload_test_reports: ${{ inputs.upload_test_reports || false }}
+      ignore_errors: ${{ inputs.ignore_errors || false }}
+      skip_list: ${{ inputs.skip_list }}
+      run_name: ${{ inputs.run_name }}
+      enable_unskip: ${{ inputs.enable_unskip || false }}
+      env_manager: conda
diff --git a/scripts/runner_default.sh b/scripts/runner_default.sh
new file mode 100644
index 0000000000..f61ba66ae8
--- /dev/null
+++ b/scripts/runner_default.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+source /opt/intel/oneapi/setvars.sh >/dev/null
+source {0}

From f73dbf0220bf39c977c23f7495feb64729b9d84f Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 14:55:08 +0200
Subject: [PATCH 003/105] Add a custom runner, r1

---
 .github/workflows/build-test-reusable.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index d4d1cf01fc..9d586e9eaa 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -78,7 +78,6 @@ jobs:
     defaults:
       run:
         shell: bash -noprofile --norc -eo pipefail scripts/runner_${{ inputs.env_manager }}.sh"
-        envinputs.env-c "source /opt/intel/oneapi/setvars.sh > /dev/null; source {0}"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4

From 21e2444cc60c51bd9d5fd09d612f58ead368d02a Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 15:03:35 +0200
Subject: [PATCH 004/105] Add a custom runner, r2

---
 .github/workflows/build-test-reusable.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 9d586e9eaa..1610f8c828 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -60,9 +60,9 @@ on:
         description: Environment manager
         type: choice
         options:
-        - default
-        - conda
-        - nobasekit      
+          - default
+          - conda
+          - nobasekit      
         default: default
 
 permissions: read-all

From e128768d8cc7a1c53ef22d4015d092509884cc34 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 15:10:13 +0200
Subject: [PATCH 005/105] Add a custom runner, r3

---
 .github/workflows/build-test-reusable.yml  | 6 +++---
 scripts/{runner_default.sh => run_base.sh} | 0
 2 files changed, 3 insertions(+), 3 deletions(-)
 rename scripts/{runner_default.sh => run_base.sh} (100%)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 1610f8c828..ddb5b46541 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -58,12 +58,12 @@ on:
         default: runner-0.0.19
       env_manager:
         description: Environment manager
+        default: base
         type: choice
         options:
-          - default
+          - base
           - conda
           - nobasekit      
-        default: default
 
 permissions: read-all
 
@@ -77,7 +77,7 @@ jobs:
     runs-on: ${{ fromJson(inputs.runner_label && format('["{0}"]', inputs.runner_label) || format('["{0}", "{1}", "{2}"]', inputs.device, inputs.driver_version, inputs.runner_version)) }}
     defaults:
       run:
-        shell: bash -noprofile --norc -eo pipefail scripts/runner_${{ inputs.env_manager }}.sh"
+        shell: bash -noprofile --norc -eo pipefail scripts/run_${{ inputs.env_manager }}.sh"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
diff --git a/scripts/runner_default.sh b/scripts/run_base.sh
similarity index 100%
rename from scripts/runner_default.sh
rename to scripts/run_base.sh

From b983d1b68d3ee0d852904af11b5178340eb1a045 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 15:22:16 +0200
Subject: [PATCH 006/105] Add a custom runner, r4

---
 .github/workflows/build-test-reusable.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index ddb5b46541..c49ae1c091 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -59,11 +59,11 @@ on:
       env_manager:
         description: Environment manager
         default: base
-        type: choice
-        options:
-          - base
-          - conda
-          - nobasekit      
+        type: sting
+#        options:
+#         - base
+#          - conda
+#          - nobasekit      
 
 permissions: read-all
 

From 763e992e3efc3a6097c04ad61d35cc5450192e4c Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 15:27:06 +0200
Subject: [PATCH 007/105] Add a custom runner, r5

---
 .github/workflows/build-test-reusable.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index c49ae1c091..e32e09a23b 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -59,7 +59,7 @@ on:
       env_manager:
         description: Environment manager
         default: base
-        type: sting
+        type: string
 #        options:
 #         - base
 #          - conda

From b96a7b87a282cef463f3d0310c43d036e124c96e Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 15:29:48 +0200
Subject: [PATCH 008/105] Add a custom runner, r6

---
 .github/workflows/build-test-reusable.yml | 2 +-
 scripts/run_base.sh                       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index e32e09a23b..a6227d5792 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -77,7 +77,7 @@ jobs:
     runs-on: ${{ fromJson(inputs.runner_label && format('["{0}"]', inputs.runner_label) || format('["{0}", "{1}", "{2}"]', inputs.device, inputs.driver_version, inputs.runner_version)) }}
     defaults:
       run:
-        shell: bash -noprofile --norc -eo pipefail scripts/run_${{ inputs.env_manager }}.sh"
+        shell: bash -noprofile --norc -eo pipefail scripts/run_${{ inputs.env_manager }}.sh {0}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
diff --git a/scripts/run_base.sh b/scripts/run_base.sh
index f61ba66ae8..70c65e8ff2 100644
--- a/scripts/run_base.sh
+++ b/scripts/run_base.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
 
 source /opt/intel/oneapi/setvars.sh >/dev/null
-source {0}
+source $1

From 1e566e4cc355486694ea3fce6c555fc2c889b55b Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 15:31:41 +0200
Subject: [PATCH 009/105] Add a custom runner, r7

---
 .github/workflows/build-test-reusable.yml | 10 +++++-----
 .github/workflows/build-test.yml          |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index a6227d5792..1039a934d7 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -59,11 +59,11 @@ on:
       env_manager:
         description: Environment manager
         default: base
-        type: string
-#        options:
-#         - base
-#          - conda
-#          - nobasekit      
+        type: choice
+        options:
+          - base
+          - conda
+          - nobasekit
 
 permissions: read-all
 
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
index 512162c2aa..7f23130023 100644
--- a/.github/workflows/build-test.yml
+++ b/.github/workflows/build-test.yml
@@ -44,7 +44,7 @@ on:
     branches:
       - main
   push:
-# FIXME    
+# FIXME
 #    branches:
 #      - main
 

From c12fe51d5e457df1f6df1430b6983172a466be7f Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 15:58:11 +0200
Subject: [PATCH 010/105] Add conda runner

---
 .github/workflows/build-test-reusable.yml |  2 +-
 scripts/run_conda.sh                      | 20 ++++++++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)
 create mode 100644 scripts/run_conda.sh

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 1039a934d7..e754527de3 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -59,7 +59,7 @@ on:
       env_manager:
         description: Environment manager
         default: base
-        type: choice
+        type: string
         options:
           - base
           - conda
diff --git a/scripts/run_conda.sh b/scripts/run_conda.sh
new file mode 100644
index 0000000000..a2d54f4456
--- /dev/null
+++ b/scripts/run_conda.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -vx
+
+install_conda() {
+    conda create -n triton --override-channels -c conda-forge python=${{ matrix.python }}.*
+    conda env update -f scripts/triton.yml
+    find /opt/intel/oneapi/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
+    ln -snf /usr/include/level_zero $HOME/miniforge3/envs/triton/bin/../x86_64-conda-linux-gnu/sysroot/usr/include/level_zero
+    find /usr -name libze_\* -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
+    conda info
+    conda list -n triton
+}
+
+
+test -d $HOME/miniforge3/envs/triton || install_conda
+
+source /opt/intel/oneapi/setvars.sh >/dev/null
+conda run --no-capture-output -n triton bash $1
+

From f9b84f1c91ef05acf7e253c9d1bfec40510a389b Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 16:05:19 +0200
Subject: [PATCH 011/105] Add conda runner, r1

---
 .github/workflows/build-test-reusable.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index e754527de3..dbcb5fcb19 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -60,10 +60,6 @@ on:
         description: Environment manager
         default: base
         type: string
-        options:
-          - base
-          - conda
-          - nobasekit
 
 permissions: read-all
 

From 0ff7ff1d4508a580aa921534fa0f6babf2c13f9b Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 16:18:14 +0200
Subject: [PATCH 012/105] Add conda runner, r

---
 .github/workflows/build-test.yml       | 5 ++---
 .github/workflows/conda-build-test.yml | 3 +--
 scripts/run_conda.sh                   | 1 -
 3 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
index 7f23130023..d3d9b29b56 100644
--- a/.github/workflows/build-test.yml
+++ b/.github/workflows/build-test.yml
@@ -44,9 +44,8 @@ on:
     branches:
       - main
   push:
-# FIXME
-#    branches:
-#      - main
+    branches:
+      - main
 
 permissions: read-all
 
diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml
index dbc3f76135..104283d5c3 100644
--- a/.github/workflows/conda-build-test.yml
+++ b/.github/workflows/conda-build-test.yml
@@ -43,9 +43,8 @@ on:
   pull_request:
     branches:
       - main
+  #FIXME      
   push:
-    branches:
-      - main
 
 permissions: read-all
 
diff --git a/scripts/run_conda.sh b/scripts/run_conda.sh
index a2d54f4456..346a3ce30f 100644
--- a/scripts/run_conda.sh
+++ b/scripts/run_conda.sh
@@ -17,4 +17,3 @@ test -d $HOME/miniforge3/envs/triton || install_conda
 
 source /opt/intel/oneapi/setvars.sh >/dev/null
 conda run --no-capture-output -n triton bash $1
-

From 4eaac5e0783b76eda064828e394fa27d51925394 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 16:25:22 +0200
Subject: [PATCH 013/105] Add conda runner, r2

---
 .github/workflows/build-test-reusable.yml | 3 ++-
 scripts/run_base.sh                       | 2 +-
 scripts/run_conda.sh                      | 7 +++++--
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index dbcb5fcb19..7e2d33730e 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -73,7 +73,7 @@ jobs:
     runs-on: ${{ fromJson(inputs.runner_label && format('["{0}"]', inputs.runner_label) || format('["{0}", "{1}", "{2}"]', inputs.device, inputs.driver_version, inputs.runner_version)) }}
     defaults:
       run:
-        shell: bash -noprofile --norc -eo pipefail scripts/run_${{ inputs.env_manager }}.sh {0}
+        shell: bash -noprofile --norc -eo pipefail scripts/run_${{ inputs.env_manager }}.sh --python ${{ inputs.python_version }} {0}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -95,6 +95,7 @@ jobs:
           key: pip-${{ inputs.python_version }}-${{ hashFiles('python/pyproject.toml', 'python/setup.py') }}-${{ env.CACHE_NUMBER }}
 
       - name: Install Python ${{ inputs.python_version }}
+        if: env_manager == "base"
         uses: actions/setup-python@v5
         with:
           python-version: ${{ inputs.python_version }}
diff --git a/scripts/run_base.sh b/scripts/run_base.sh
index 70c65e8ff2..e385404f6c 100644
--- a/scripts/run_base.sh
+++ b/scripts/run_base.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
 
 source /opt/intel/oneapi/setvars.sh >/dev/null
-source $1
+source $3
diff --git a/scripts/run_conda.sh b/scripts/run_conda.sh
index 346a3ce30f..8460ec2fe0 100644
--- a/scripts/run_conda.sh
+++ b/scripts/run_conda.sh
@@ -2,8 +2,11 @@
 
 set -vx
 
+python_version=$2
+script_name=$3
+
 install_conda() {
-    conda create -n triton --override-channels -c conda-forge python=${{ matrix.python }}.*
+    conda create -n triton --override-channels -c conda-forge python=$python_version.*
     conda env update -f scripts/triton.yml
     find /opt/intel/oneapi/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
     ln -snf /usr/include/level_zero $HOME/miniforge3/envs/triton/bin/../x86_64-conda-linux-gnu/sysroot/usr/include/level_zero
@@ -16,4 +19,4 @@ install_conda() {
 test -d $HOME/miniforge3/envs/triton || install_conda
 
 source /opt/intel/oneapi/setvars.sh >/dev/null
-conda run --no-capture-output -n triton bash $1
+conda run --no-capture-output -n triton bash $script_name

From f394e60bddc20de96f374009431d9c7628e16730 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 16:27:02 +0200
Subject: [PATCH 014/105] Add conda runner, r3

---
 .github/workflows/build-test-reusable.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 7e2d33730e..e5e3fa5e3a 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -95,7 +95,7 @@ jobs:
           key: pip-${{ inputs.python_version }}-${{ hashFiles('python/pyproject.toml', 'python/setup.py') }}-${{ env.CACHE_NUMBER }}
 
       - name: Install Python ${{ inputs.python_version }}
-        if: env_manager == "base"
+        if: inputs.env_manager == "base"
         uses: actions/setup-python@v5
         with:
           python-version: ${{ inputs.python_version }}

From c2f4fb1f636bf668bd7587fe783a4e8a5965da8c Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 16:28:19 +0200
Subject: [PATCH 015/105] Add conda runner, r4

---
 .github/workflows/build-test-reusable.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index e5e3fa5e3a..ae95c8d47a 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -95,7 +95,7 @@ jobs:
           key: pip-${{ inputs.python_version }}-${{ hashFiles('python/pyproject.toml', 'python/setup.py') }}-${{ env.CACHE_NUMBER }}
 
       - name: Install Python ${{ inputs.python_version }}
-        if: inputs.env_manager == "base"
+        if: inputs.env_manager == 'base'
         uses: actions/setup-python@v5
         with:
           python-version: ${{ inputs.python_version }}

From b5a092cdafc3f7626df82b02b42111d0f6418a23 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 16:32:12 +0200
Subject: [PATCH 016/105] Add conda runner, r

---
 .github/workflows/conda-build-test.yml | 2 +-
 scripts/run_conda.sh                   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml
index 104283d5c3..2bda970b3f 100644
--- a/.github/workflows/conda-build-test.yml
+++ b/.github/workflows/conda-build-test.yml
@@ -130,7 +130,7 @@ jobs:
       python_version: ${{ matrix.python }}
       upload_test_reports: ${{ inputs.upload_test_reports || false }}
       ignore_errors: ${{ inputs.ignore_errors || false }}
-      skip_list: ${{ inputs.skip_list }}
+      skip_list: conda
       run_name: ${{ inputs.run_name }}
       enable_unskip: ${{ inputs.enable_unskip || false }}
       env_manager: conda
diff --git a/scripts/run_conda.sh b/scripts/run_conda.sh
index 8460ec2fe0..91e7cff1ab 100644
--- a/scripts/run_conda.sh
+++ b/scripts/run_conda.sh
@@ -15,7 +15,7 @@ install_conda() {
     conda list -n triton
 }
 
-
+export PATH=$HOME/miniforge3/bin:$PATH
 test -d $HOME/miniforge3/envs/triton || install_conda
 
 source /opt/intel/oneapi/setvars.sh >/dev/null

From 790e4bf429839f6ffc20ab9a95e440efe9b9fbeb Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 16:39:20 +0200
Subject: [PATCH 017/105] Add conda runner, r

---
 .github/workflows/build-test-reusable.yml |  2 +-
 scripts/run_base.sh                       |  5 ++++-
 scripts/run_conda.sh                      | 12 ++++++------
 scripts/run_util.sh                       | 20 ++++++++++++++++++++
 4 files changed, 31 insertions(+), 8 deletions(-)
 create mode 100644 scripts/run_util.sh

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index ae95c8d47a..7352d8b3f3 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -73,7 +73,7 @@ jobs:
     runs-on: ${{ fromJson(inputs.runner_label && format('["{0}"]', inputs.runner_label) || format('["{0}", "{1}", "{2}"]', inputs.device, inputs.driver_version, inputs.runner_version)) }}
     defaults:
       run:
-        shell: bash -noprofile --norc -eo pipefail scripts/run_${{ inputs.env_manager }}.sh --python ${{ inputs.python_version }} {0}
+        shell: bash -noprofile --norc -eo pipefail scripts/run_${{ inputs.env_manager }}.sh --python-version ${{ inputs.python_version }} {0}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
diff --git a/scripts/run_base.sh b/scripts/run_base.sh
index e385404f6c..7379f27053 100644
--- a/scripts/run_base.sh
+++ b/scripts/run_base.sh
@@ -1,4 +1,7 @@
 #!/bin/bash
 
 source /opt/intel/oneapi/setvars.sh >/dev/null
-source $3
+
+script_dir=$(dirname "$0")
+source "$script_dir/run_util.sh"
+source "$script_name"
diff --git a/scripts/run_conda.sh b/scripts/run_conda.sh
index 91e7cff1ab..39df24a4e3 100644
--- a/scripts/run_conda.sh
+++ b/scripts/run_conda.sh
@@ -2,9 +2,6 @@
 
 set -vx
 
-python_version=$2
-script_name=$3
-
 install_conda() {
     conda create -n triton --override-channels -c conda-forge python=$python_version.*
     conda env update -f scripts/triton.yml
@@ -15,8 +12,11 @@ install_conda() {
     conda list -n triton
 }
 
-export PATH=$HOME/miniforge3/bin:$PATH
-test -d $HOME/miniforge3/envs/triton || install_conda
+script_dir=$(dirname "$0")
+source "$script_dir/run_util.sh"
+
+export PATH="$HOME/miniforge3/bin:$PATH"
+test -d "$HOME/miniforge3/envs/triton" || install_conda
 
 source /opt/intel/oneapi/setvars.sh >/dev/null
-conda run --no-capture-output -n triton bash $script_name
+conda run --no-capture-output -n triton bash "$script_name"
diff --git a/scripts/run_util.sh b/scripts/run_util.sh
new file mode 100644
index 0000000000..e1ad67811b
--- /dev/null
+++ b/scripts/run_util.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+while [ -v 1 ]; do
+  case "$1" in
+    --python-version)
+      python_version=$2
+      shift 2
+      ;;
+    --unit)
+      TEST_UNIT=true
+      shift
+      ;;
+    --core)
+      TEST_CORE=true
+      shift
+    *)
+      script_name=$1
+      shift
+  esac
+done

From f95f102a0a53e0583e51667b7bd2cfe16bdc2bd1 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 16:40:16 +0200
Subject: [PATCH 018/105] Add conda runner, r

---
 scripts/run_util.sh | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/scripts/run_util.sh b/scripts/run_util.sh
index e1ad67811b..ed39558603 100644
--- a/scripts/run_util.sh
+++ b/scripts/run_util.sh
@@ -6,15 +6,9 @@ while [ -v 1 ]; do
       python_version=$2
       shift 2
       ;;
-    --unit)
-      TEST_UNIT=true
-      shift
-      ;;
-    --core)
-      TEST_CORE=true
-      shift
     *)
       script_name=$1
       shift
+      ;;
   esac
 done

From 3e8944fb44916a92371091da5a09c71d00cd8288 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 16:43:39 +0200
Subject: [PATCH 019/105] Add conda runner, r

---
 scripts/run_conda.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/run_conda.sh b/scripts/run_conda.sh
index 39df24a4e3..71562cac3a 100644
--- a/scripts/run_conda.sh
+++ b/scripts/run_conda.sh
@@ -12,11 +12,12 @@ install_conda() {
     conda list -n triton
 }
 
+source /opt/intel/oneapi/setvars.sh >/dev/null
+
 script_dir=$(dirname "$0")
 source "$script_dir/run_util.sh"
 
 export PATH="$HOME/miniforge3/bin:$PATH"
 test -d "$HOME/miniforge3/envs/triton" || install_conda
 
-source /opt/intel/oneapi/setvars.sh >/dev/null
 conda run --no-capture-output -n triton bash "$script_name"

From 5a8d34e8ec4f9f9def74dceba7f99dced053cd33 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 18:29:10 +0200
Subject: [PATCH 020/105] Add conda runner, r

---
 .github/workflows/build-test-reusable.yml | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 7352d8b3f3..afb5afc808 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -78,6 +78,16 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Load conda cache
+        if: inputs.env_manager != 'base'
+        id: conda-cache
+        uses: ./.github/actions/load
+        env:
+          CACHE_NUMBER: 5
+        with:
+          path: $HOME/miniforge3/envs/triton
+          key: conda-py${{ matrix.python }}-${{ hashFiles('scripts/triton.yml', 'python/pyproject.toml', 'python/setup.py', '.github/pins/ipex.txt', '.github/pins/pytorch.txt') }}-${{ env.CACHE_NUMBER }}
+
       - name: Print inputs
         run: |
           cat <<EOF
@@ -85,6 +95,7 @@ jobs:
           EOF
 
       - name: Load pip cache
+        if: inputs.env_manager == 'base'
         id: pip-cache
         uses: ./.github/actions/load
         env:
@@ -101,6 +112,7 @@ jobs:
           python-version: ${{ inputs.python_version }}
 
       - name: Setup PyTorch
+        if: inputs.env_manager == 'base'
         uses: ./.github/actions/setup-pytorch
         with:
           repository: pytorch/pytorch
@@ -112,6 +124,7 @@ jobs:
           pip install defusedxml
 
       - name: Setup Triton
+        if: inputs.env_manager == 'base'
         uses: ./.github/actions/setup-triton
         with:
           build_llvm: ${{ inputs.build_llvm }}

From 388ab7cb8bcec3f43235d0cf332d935fac7c9d44 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 18:38:16 +0200
Subject: [PATCH 021/105] Add conda runner, r

---
 .github/workflows/build-test-reusable.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index afb5afc808..5b4c45a029 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -129,6 +129,14 @@ jobs:
         with:
           build_llvm: ${{ inputs.build_llvm }}
 
+      - name: Setup Triton
+        if: inputs.env_manager != 'base'
+        run: |
+          export DEBUG=1
+          cd python
+          conda run --no-capture-output -n triton pip install pybind11
+          conda run --no-capture-output -n triton pip install --no-build-isolation -e '.[build,tests,tutorials]'
+
       - name: Create test-triton command line
         run: |
           if [[ -n "${{ inputs.skip_list }}" ]]; then

From 75c1290010b6d066e1e213609e5a851d056be3bd Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 7 Oct 2024 19:32:19 +0200
Subject: [PATCH 022/105] Add conda runner, r

---
 .github/workflows/build-test-reusable.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 5b4c45a029..90cf9436d5 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -119,6 +119,11 @@ jobs:
           ref: ${{ inputs.pytorch_ref }}
           mode: ${{ inputs.pytorch_mode }}
 
+      - name: Setup PyTorch
+        if: inputs.env_manager != 'base'
+        run: |
+          pip install torch --index-url https://download.pytorch.org/whl/nightly/xpu
+
       - name: Install pass_rate dependencies
         run: |
           pip install defusedxml

From bfe878ec99e9e73f52227263e53a7b9e77f56e60 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Tue, 8 Oct 2024 13:45:03 +0200
Subject: [PATCH 023/105] Update conda skiplist

---
 scripts/skiplist/conda/language.txt | 62 +++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/scripts/skiplist/conda/language.txt b/scripts/skiplist/conda/language.txt
index cdac848de1..45726eab99 100644
--- a/scripts/skiplist/conda/language.txt
+++ b/scripts/skiplist/conda/language.txt
@@ -161,3 +161,65 @@ test/unit/language/test_core.py::test_dot3d[8-8-64-64-64-32-32-float16-float16]
 test/unit/language/test_core.py::test_dot3d[8-8-64-64-64-32-32-float16-float32]
 test/unit/language/test_core.py::test_dot3d[8-8-64-64-64-32-32-float32-float32]
 test/unit/language/test_core.py::test_dot3d[8-8-64-64-64-32-32-int8-int8]
+# Other failures
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint8-%]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint16-%]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint32-%]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint64-%]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint8-%]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint16-%]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint32-%]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint64-%]
+test/unit/language/test_compile_errors.py::test_where_warning
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint8-%]
+test/unit/language/test_compile_errors.py::test_fp8_support[dtype1]
+test/unit/language/test_compile_errors.py::test_fp8_support[dtype3]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint16-%]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint32-%]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint64-%]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint8-%]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint16-%]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint32-%]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint64-%]
+test/unit/language/test_conversions.py::test_typeconvert_upcast[float8e4b8-float32]
+test/unit/language/test_conversions.py::test_typeconvert_upcast[float8e4b8-float16]
+test/unit/language/test_conversions.py::test_typeconvert_upcast[float8e5b16-float32]
+test/unit/language/test_conversions.py::test_typeconvert_upcast[float8e5b16-float16]
+test/unit/language/test_core.py::test_floordiv[1-uint8-uint8]
+test/unit/language/test_core.py::test_floordiv[1-uint8-uint16]
+test/unit/language/test_core.py::test_floordiv[1-uint8-uint32]
+test/unit/language/test_core.py::test_floordiv[1-uint8-uint64]
+test/unit/language/test_core.py::test_floordiv[1-uint16-uint8]
+test/unit/language/test_core.py::test_floordiv[1-uint16-uint16]
+test/unit/language/test_core.py::test_floordiv[1-uint16-uint32]
+test/unit/language/test_core.py::test_floordiv[1-uint16-uint64]
+test/unit/language/test_core.py::test_floordiv[1-uint32-uint8]
+test/unit/language/test_core.py::test_floordiv[1-uint32-uint16]
+test/unit/language/test_core.py::test_floordiv[1-uint32-uint32]
+test/unit/language/test_core.py::test_floordiv[1-uint32-uint64]
+test/unit/language/test_core.py::test_floordiv[1-uint64-uint8]
+test/unit/language/test_core.py::test_floordiv[1-uint64-uint16]
+test/unit/language/test_core.py::test_floordiv[1-uint64-uint32]
+test/unit/language/test_core.py::test_floordiv[1-uint64-uint64]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint8-/]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint16-/]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint32-/]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint64-/]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint8-/]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint16-/]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint32-/]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint64-/]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint8-/]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint16-/]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint32-/]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint64-/]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint8-/]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint16-/]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint32-/]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint64-/]
+test/unit/language/test_core.py::test_const[ternary-False-True]
+test/unit/language/test_core.py::test_const[ternary-False-False]
+test/unit/language/test_core.py::test_const[if-False-True]
+test/unit/language/test_core.py::test_const[if-False-False]
+test/unit/language/test_core.py::test_load_cache_modifier[.cv]
+test/unit/language/test_core.py::test_unroll_attr

From 7e5b3e444816bdc621ff624de812b83c20ba4b53 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Tue, 8 Oct 2024 13:58:32 +0200
Subject: [PATCH 024/105] Add nobasekit run

---
 .github/workflows/conda-build-test.yml      |   2 +-
 .github/workflows/no-basekit-build-test.yml | 217 ++++++++++----------
 scripts/run_conda.sh                        |   1 +
 scripts/run_util.sh                         |   5 +
 4 files changed, 115 insertions(+), 110 deletions(-)

diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml
index 2bda970b3f..fea2108cc6 100644
--- a/.github/workflows/conda-build-test.yml
+++ b/.github/workflows/conda-build-test.yml
@@ -1,4 +1,4 @@
-name: Build and test
+name: Conda test
 run-name: ${{ inputs.run_name }}
 
 on:
diff --git a/.github/workflows/no-basekit-build-test.yml b/.github/workflows/no-basekit-build-test.yml
index dfa0313cbb..ecb5ab8c20 100644
--- a/.github/workflows/no-basekit-build-test.yml
+++ b/.github/workflows/no-basekit-build-test.yml
@@ -1,137 +1,136 @@
-name: No-basekit build and test
+name: Nobasekit test
+run-name: ${{ inputs.run_name }}
 
 on:
   workflow_dispatch:
-  schedule:
-    - cron: "5 2 * * *"
+    inputs:
+      runner_label:
+        description: Runner label, keep empty for default
+        type: string
+        default: ""
+      pytorch_ref:
+        description: PyTorch ref, keep empty for default
+        type: string
+        default: ""
+      pytorch_mode:
+        description: PyTorch mode, source or wheels
+        type: choice
+        options:
+          - source
+          - wheels
+        default: source
+      upload_test_reports:
+        description: Upload test reports
+        type: boolean
+        default: false
+      ignore_errors:
+        description: Ignore test errors
+        type: boolean
+        default: false
+      skip_list:
+        description: Skip list
+        type: string
+        default: ""
+      run_name:
+        description: Custom run name
+        type: string
+        default: "Build and test"
+      enable_unskip:
+        description: Ignore pytest.skip
+        type: boolean
+        default: false
+
+  pull_request:
+    branches:
+      - main
+  #FIXME      
+  push:
 
 permissions: read-all
 
-env:
-  TRITON_DISABLE_LINE_INFO: 1
-
 jobs:
-
-  integration-tests:
-    name: Integration tests
-    runs-on:
-      - max1100
-      - rolling
-      - runner-0.0.19
-    strategy:
-      matrix:
-        python: ${{ github.ref_name == 'main' && fromJson('["3.9", "3.10", "3.11"]') || fromJson('["3.9"]') }}
-    defaults:
-      run:
-        shell: bash -noprofile --norc -eo pipefail -c "source {0}"
+  pre-commit:
+    name: Pre-commit checks
+    runs-on: Linux
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Calculate PATH
-        run: |
-          echo $HOME/miniforge3/bin >>$GITHUB_PATH
-
-      - name: Load conda cache
-        id: conda-cache
+      - name: Load pip cache
+        id: pip-cache
         uses: ./.github/actions/load
         env:
-          CACHE_NUMBER: 6
+          # Increase this value to reset cache
+          CACHE_NUMBER: 2
         with:
-          path: $HOME/miniforge3/envs/triton
-          key: no-basekit-py${{ matrix.python }}-${{ hashFiles('scripts/triton.yml', 'python/pyproject.toml', 'python/setup.py', '.github/pins/ipex.txt', '.github/pins/pytorch.txt') }}-${{ env.CACHE_NUMBER }}
+          path: $HOME/.cache/pip
+          key: pip-3.10-${{ hashFiles('.pre-commit-config.yaml') }}-${{ env.CACHE_NUMBER }}
 
-      - name: Update conda env
-        if: ${{ steps.conda-cache.outputs.status == 'miss' }}
-        run: |
-          conda create -n triton --override-channels -c conda-forge python=${{ matrix.python }}.*
-          conda env update -f scripts/triton.yml
-
-          ln -snf /usr/include/level_zero $HOME/miniforge3/envs/triton/x86_64-conda-linux-gnu/sysroot/usr/include/level_zero
-          find /usr -name libze_\* -exec ln -sf {} $HOME/miniforge3/envs/triton/lib/ \;
-
-          find /opt/intel/oneapi/mkl/2025.0/lib/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
-          find /opt/intel/oneapi/compiler/2024.1/lib/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
+      - name: Install Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
 
-      - name: Calculate ENV
+      - name: Run pre-commit checks
         run: |
-          python -m venv ./.venv; source ./.venv/bin/activate
-          echo LD_LIBRARY_PATH=$HOME/miniforge3/envs/triton/lib:$LD_LIBRARY_PATH:$VIRTUAL_ENV/lib >>$GITHUB_ENV
-          echo CPATH=$CPATH:$VIRTUAL_ENV/include:$VIRTUAL_ENV/include/sycl >>$GITHUB_ENV
+          set -x
+          pip install --upgrade pre-commit
 
-      - name: Install deps
-        run: |
-          python -m venv ./.venv; source ./.venv/bin/activate
+          # TODO: ignore the first yapf failure until https://github.com/google/yapf/issues/1164 is fixed
+          python3 -m pre_commit run --all-files --verbose yapf &> /dev/null || true
+          # If first run of yapf worked and made changes reset the tree to the original state
+          git reset --hard
 
-          wget https://files.pythonhosted.org/packages/cc/1e/d74e608f0c040e4f72dbfcd3b183f39570f054d08de39cc431f153220d90/intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
-          wheel unpack intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
-          mkdir -p ./intel_sycl_rt-2024.1.2/intel_sycl_rt-2024.1.2.data/data/include
-          cp -r /opt/intel/oneapi/compiler/2024.1/include/sycl ./intel_sycl_rt-2024.*/intel_sycl_rt-2024.*.data/data/include/
-          wheel pack intel_sycl_rt-2024.1.2 --build headers_patch
-          mv intel_sycl_rt-2024.1.2-headers_patch-py2.py3-none-manylinux1_x86_64.whl intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
-          pip install --force-reinstall ./intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
-          pip install dpcpp_cpp_rt==2024.1.2
+          python3 -m pre_commit run --show-diff-on-failure --color=always --all-files --verbose
 
+      - name: Save pip cache
+        if: ${{ steps.pip-cache.outputs.status == 'miss' }}
+        uses: ./.github/actions/save
+        with:
+          path: ${{ steps.pip-cache.outputs.path }}
+          dest: ${{ steps.pip-cache.outputs.dest }}
 
-      - name: Add conda info to log
-        run: |
-          conda info
-          conda list -n triton
+  prepare:
+    name: Prepare
+    runs-on: Linux
 
-      - name: Install latest nightly wheels
-        uses: ./.github/actions/install-wheels
-        with:
-          gh_token: ${{ secrets.GITHUB_TOKEN }}
-          install_cmd: conda run --no-capture-output -n triton pip install
-          python_version: ${{ matrix.python }}
-          wheels_pattern: 'torch-*'
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
 
-      - name: Build Triton
+    steps:
+      - name: Inputs
         run: |
-          set -x
-          python -m venv ./.venv; source ./.venv/bin/activate
-          export DEBUG=1
-          cd python
-          conda run --no-capture-output -n triton pip install pybind11
-          conda run --no-capture-output -n triton pip install --no-build-isolation -e '.[build,tests,tutorials]'
+          cat <<EOF
+          ${{ toJSON(inputs) }}
+          EOF
 
-      - name: Run tests
+      - name: Matrix
+        id: matrix
         run: |
-          set -x
-          python -m venv ./.venv; source ./.venv/bin/activate
+          if [[ -n "${{ inputs.runner_label }}" ]]; then
+            matrix='{"python": ["3.9"]}'
+          else
+            matrix='{"python": ["3.9"], "driver": ["rolling", "lts"]}'
+          fi
+          echo "matrix=$matrix" | tee -a $GITHUB_OUTPUT
 
-          export TRITON_TEST_REPORTS=true
-          export TRITON_TEST_REPORTS_DIR=$HOME/reports
-          # FIXME https://github.com/intel/intel-xpu-backend-for-triton/issues/866
-          export TRITON_TEST_SKIPLIST_DIR=scripts/skiplist/no-basekit
-          conda run --no-capture-output -n triton bash -v -x scripts/test-triton.sh
+  integration-tests:
+    name: Integration tests matrix
+    needs: prepare
 
-      - name: Run E2E test
-        run: |
-          # FIXME https://github.com/intel/intel-xpu-backend-for-triton/issues/1031
-          cd ../pytorch || {
-            PYTORCH_COMMIT_ID=$(<.github/pins/pytorch.txt)
-            cd ..
-            git clone --single-branch -b dev/triton-test-3.0 --recurse-submodules https://github.com/Stonepia/pytorch.git
-            cd pytorch
-            git branch pin-branch $PYTORCH_COMMIT_ID
-            git switch pin-branch
-          }
-
-          TRANSFORMERS_VERSION="$(<.ci/docker/ci_commit_pins/huggingface.txt)"
-          conda run -n triton pip install pyyaml pandas scipy numpy psutil pyre_extensions torchrec transformers==$TRANSFORMERS_VERSION
-
-          # Set WORKSPACE for inductor_xpu_test.sh to make sure it creates "inductor_log" outside of pytorch cloned directory
-          export WORKSPACE=$GITHUB_WORKSPACE
-          # TODO: Find the fastest Hugging Face model
-          conda run --no-capture-output -n triton $GITHUB_WORKSPACE/scripts/inductor_xpu_test.sh huggingface float32 inference accuracy xpu 0 static 1 0 AlbertForMaskedLM
-          # The script above always returns 0, so we need an additional check to see if the accuracy test passed
-          cat $WORKSPACE/inductor_log/*/*/*.csv
-          grep AlbertForMaskedLM $WORKSPACE/inductor_log/*/*/*.csv | grep -q ,pass,
-
-      - name: Save conda cache
-        if: ${{ steps.conda-cache.outputs.status == 'miss' }}
-        uses: ./.github/actions/save
-        with:
-          path: ${{ steps.conda-cache.outputs.path }}
-          dest: ${{ steps.conda-cache.outputs.dest }}
+    strategy:
+      matrix: ${{ fromJson(needs.prepare.outputs.matrix) }}
+
+    uses: ./.github/workflows/build-test-reusable.yml
+    with:
+      driver_version: ${{ matrix.driver }}
+      runner_label: ${{ inputs.runner_label }}
+      pytorch_ref: ${{ inputs.pytorch_ref }}
+      pytorch_mode: ${{ inputs.pytorch_mode || 'source' }}
+      python_version: ${{ matrix.python }}
+      upload_test_reports: ${{ inputs.upload_test_reports || false }}
+      ignore_errors: ${{ inputs.ignore_errors || false }}
+      skip_list: nobasekit
+      run_name: ${{ inputs.run_name }}
+      enable_unskip: ${{ inputs.enable_unskip || false }}
+      env_manager: nobasekit
diff --git a/scripts/run_conda.sh b/scripts/run_conda.sh
index 71562cac3a..b19744d59a 100644
--- a/scripts/run_conda.sh
+++ b/scripts/run_conda.sh
@@ -19,5 +19,6 @@ source "$script_dir/run_util.sh"
 
 export PATH="$HOME/miniforge3/bin:$PATH"
 test -d "$HOME/miniforge3/envs/triton" || install_conda
+print_conda_info
 
 conda run --no-capture-output -n triton bash "$script_name"
diff --git a/scripts/run_util.sh b/scripts/run_util.sh
index ed39558603..8b0a353821 100644
--- a/scripts/run_util.sh
+++ b/scripts/run_util.sh
@@ -1,5 +1,10 @@
 #!/bin/bash
 
+print_conda_info() {
+    conda info
+    conda list -n triton
+}
+
 while [ -v 1 ]; do
   case "$1" in
     --python-version)

From 4fee3afbcebd9768f73bd0c515c8c818139872d4 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Tue, 8 Oct 2024 13:58:53 +0200
Subject: [PATCH 025/105] Add nobasekit run, t

---
 scripts/run_nobasekit.sh | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 scripts/run_nobasekit.sh

diff --git a/scripts/run_nobasekit.sh b/scripts/run_nobasekit.sh
new file mode 100644
index 0000000000..06856204cd
--- /dev/null
+++ b/scripts/run_nobasekit.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -vx
+
+install_conda() {
+    conda create -n triton --override-channels -c conda-forge python=$python_version.*
+    conda env update -f scripts/triton.yml
+
+    ln -snf /usr/include/level_zero $HOME/miniforge3/envs/triton/x86_64-conda-linux-gnu/sysroot/usr/include/level_zero
+    find /usr -name libze_\* -exec ln -sf {} $HOME/miniforge3/envs/triton/lib/ \;
+
+    find /opt/intel/oneapi/mkl/2025.0/lib/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
+    find /opt/intel/oneapi/compiler/2024.1/lib/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
+
+    wget https://files.pythonhosted.org/packages/cc/1e/d74e608f0c040e4f72dbfcd3b183f39570f054d08de39cc431f153220d90/intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
+    wheel unpack intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
+    mkdir -p ./intel_sycl_rt-2024.1.2/intel_sycl_rt-2024.1.2.data/data/include
+    cp -r /opt/intel/oneapi/compiler/2024.1/include/sycl ./intel_sycl_rt-2024.*/intel_sycl_rt-2024.*.data/data/include/
+    wheel pack intel_sycl_rt-2024.1.2 --build headers_patch
+    mv intel_sycl_rt-2024.1.2-headers_patch-py2.py3-none-manylinux1_x86_64.whl intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
+    pip install --force-reinstall ./intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
+    pip install dpcpp_cpp_rt==2024.1.2
+}
+
+script_dir=$(dirname "$0")
+source "$script_dir/run_util.sh"
+
+export PATH="$HOME/miniforge3/bin:$PATH"
+test -d "$HOME/miniforge3/envs/triton" || install_conda
+print_conda_info
+
+python -m venv ./.venv; source ./.venv/bin/activate
+export LD_LIBRARY_PATH=$HOME/miniforge3/envs/triton/lib:$LD_LIBRARY_PATH:$VIRTUAL_ENV/lib
+export CPATH=$CPATH:$VIRTUAL_ENV/include:$VIRTUAL_ENV/include/sycl
+
+conda run --no-capture-output -n triton bash "$script_name"

From bb33c739d9a0851a4fb8b0308ae2588dfff8230b Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Tue, 8 Oct 2024 15:17:24 +0200
Subject: [PATCH 026/105] Try ignore error testrun

---
 .github/workflows/build-test-reusable.yml     |  14 +-
 .github/workflows/conda-build-test.yml        |   3 +-
 .github/workflows/no-basekit-build-test.yml   |   9 +-
 scripts/run_conda.sh                          |   4 +-
 .../{run_nobasekit.sh => run_no_basekit.sh}   |   0
 scripts/skiplist/conda/language.txt           | 225 ------------------
 scripts/skiplist/conda/subprocess.txt         |   9 -
 scripts/skiplist/no-basekit/language.txt      | 162 -------------
 scripts/skiplist/no-basekit/regression.txt    |   6 -
 scripts/skiplist/no-basekit/subprocess.txt    |   9 -
 scripts/skiplist/no-basekit/tutorials.txt     |   4 -
 11 files changed, 16 insertions(+), 429 deletions(-)
 rename scripts/{run_nobasekit.sh => run_no_basekit.sh} (100%)
 delete mode 100644 scripts/skiplist/conda/language.txt
 delete mode 100644 scripts/skiplist/conda/subprocess.txt
 delete mode 100644 scripts/skiplist/no-basekit/language.txt
 delete mode 100644 scripts/skiplist/no-basekit/regression.txt
 delete mode 100644 scripts/skiplist/no-basekit/subprocess.txt
 delete mode 100644 scripts/skiplist/no-basekit/tutorials.txt

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 90cf9436d5..66653a7c4e 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -78,6 +78,12 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Print inputs
+        run: |
+          cat <<EOF
+          ${{ toJSON(inputs) }}
+          EOF
+
       - name: Load conda cache
         if: inputs.env_manager != 'base'
         id: conda-cache
@@ -86,13 +92,7 @@ jobs:
           CACHE_NUMBER: 5
         with:
           path: $HOME/miniforge3/envs/triton
-          key: conda-py${{ matrix.python }}-${{ hashFiles('scripts/triton.yml', 'python/pyproject.toml', 'python/setup.py', '.github/pins/ipex.txt', '.github/pins/pytorch.txt') }}-${{ env.CACHE_NUMBER }}
-
-      - name: Print inputs
-        run: |
-          cat <<EOF
-          ${{ toJSON(inputs) }}
-          EOF
+          key: conda-${{ inputs.env_manager }}-py${{ matrix.python }}-${{ hashFiles('scripts/triton.yml', 'python/pyproject.toml', 'python/setup.py') }}-${{ env.CACHE_NUMBER }}
 
       - name: Load pip cache
         if: inputs.env_manager == 'base'
diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml
index fea2108cc6..a0d7bb7ca2 100644
--- a/.github/workflows/conda-build-test.yml
+++ b/.github/workflows/conda-build-test.yml
@@ -129,7 +129,8 @@ jobs:
       pytorch_mode: ${{ inputs.pytorch_mode || 'source' }}
       python_version: ${{ matrix.python }}
       upload_test_reports: ${{ inputs.upload_test_reports || false }}
-      ignore_errors: ${{ inputs.ignore_errors || false }}
+      ignore_errors: true
+      #FIXME${{ inputs.ignore_errors || false }}
       skip_list: conda
       run_name: ${{ inputs.run_name }}
       enable_unskip: ${{ inputs.enable_unskip || false }}
diff --git a/.github/workflows/no-basekit-build-test.yml b/.github/workflows/no-basekit-build-test.yml
index ecb5ab8c20..1bf8f94577 100644
--- a/.github/workflows/no-basekit-build-test.yml
+++ b/.github/workflows/no-basekit-build-test.yml
@@ -1,4 +1,4 @@
-name: Nobasekit test
+name: No-Basekit Test
 run-name: ${{ inputs.run_name }}
 
 on:
@@ -129,8 +129,9 @@ jobs:
       pytorch_mode: ${{ inputs.pytorch_mode || 'source' }}
       python_version: ${{ matrix.python }}
       upload_test_reports: ${{ inputs.upload_test_reports || false }}
-      ignore_errors: ${{ inputs.ignore_errors || false }}
-      skip_list: nobasekit
+      ignore_errors: true
+      #FIXME${{ inputs.ignore_errors || false }}
+      skip_list: no-basekit
       run_name: ${{ inputs.run_name }}
       enable_unskip: ${{ inputs.enable_unskip || false }}
-      env_manager: nobasekit
+      env_manager: no_basekit
diff --git a/scripts/run_conda.sh b/scripts/run_conda.sh
index b19744d59a..40f2100d98 100644
--- a/scripts/run_conda.sh
+++ b/scripts/run_conda.sh
@@ -1,7 +1,5 @@
 #!/bin/bash
 
-set -vx
-
 install_conda() {
     conda create -n triton --override-channels -c conda-forge python=$python_version.*
     conda env update -f scripts/triton.yml
@@ -14,6 +12,8 @@ install_conda() {
 
 source /opt/intel/oneapi/setvars.sh >/dev/null
 
+set -vx
+
 script_dir=$(dirname "$0")
 source "$script_dir/run_util.sh"
 
diff --git a/scripts/run_nobasekit.sh b/scripts/run_no_basekit.sh
similarity index 100%
rename from scripts/run_nobasekit.sh
rename to scripts/run_no_basekit.sh
diff --git a/scripts/skiplist/conda/language.txt b/scripts/skiplist/conda/language.txt
deleted file mode 100644
index 45726eab99..0000000000
--- a/scripts/skiplist/conda/language.txt
+++ /dev/null
@@ -1,225 +0,0 @@
-# https://github.com/intel/intel-xpu-backend-for-triton/issues/1434
-test/unit/language/test_core.py::test_precise_math[1-tl.math.sqrt_rn(x)-tl.math.sqrt(x.to(tl.float64)).to(tl.float32)]
-test/unit/language/test_core.py::test_dot3d[1-1-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-1-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-1-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-1-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[1-1-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-1-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-1-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-1-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[1-16-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-16-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-16-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-16-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[1-16-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-16-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-16-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-16-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[1-2-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-2-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-2-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-2-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[1-2-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-2-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-2-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-2-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[1-4-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-4-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-4-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-4-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[1-4-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-4-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-4-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-4-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[1-8-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-8-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-8-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-8-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[1-8-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-8-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-8-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-8-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-1-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-1-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-1-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-1-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-1-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-1-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-1-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-1-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-16-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-16-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-16-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-16-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-16-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-16-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-16-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-16-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-2-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-2-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-2-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-2-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-2-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-2-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-2-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-2-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-4-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-4-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-4-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-4-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-4-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-4-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-4-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-4-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-8-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-8-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-8-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-8-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-8-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-8-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-8-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-8-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-1-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-1-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-1-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-1-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-1-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-1-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-1-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-1-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-16-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-16-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-16-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-16-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-16-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-16-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-16-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-16-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-2-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-2-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-2-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-2-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-2-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-2-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-2-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-2-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-4-128-128-64-64-64-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-4-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-4-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-4-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-4-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-4-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-4-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-4-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-4-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-8-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-8-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-8-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-8-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-8-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-8-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-8-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-8-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-1-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-1-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-1-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-1-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-1-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-1-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-1-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-1-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-16-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-16-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-16-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-16-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-16-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-16-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-16-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-16-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-2-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-2-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-2-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-2-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-2-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-2-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-2-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-2-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-4-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-4-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-4-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-4-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-4-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-4-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-4-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-4-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-8-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-8-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-8-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-8-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-8-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-8-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-8-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-8-64-64-64-32-32-int8-int8]
-# Other failures
-test/unit/language/test_core.py::test_bin_op[1-uint8-uint8-%]
-test/unit/language/test_core.py::test_bin_op[1-uint8-uint16-%]
-test/unit/language/test_core.py::test_bin_op[1-uint8-uint32-%]
-test/unit/language/test_core.py::test_bin_op[1-uint8-uint64-%]
-test/unit/language/test_core.py::test_bin_op[1-uint16-uint8-%]
-test/unit/language/test_core.py::test_bin_op[1-uint16-uint16-%]
-test/unit/language/test_core.py::test_bin_op[1-uint16-uint32-%]
-test/unit/language/test_core.py::test_bin_op[1-uint16-uint64-%]
-test/unit/language/test_compile_errors.py::test_where_warning
-test/unit/language/test_core.py::test_bin_op[1-uint32-uint8-%]
-test/unit/language/test_compile_errors.py::test_fp8_support[dtype1]
-test/unit/language/test_compile_errors.py::test_fp8_support[dtype3]
-test/unit/language/test_core.py::test_bin_op[1-uint32-uint16-%]
-test/unit/language/test_core.py::test_bin_op[1-uint32-uint32-%]
-test/unit/language/test_core.py::test_bin_op[1-uint32-uint64-%]
-test/unit/language/test_core.py::test_bin_op[1-uint64-uint8-%]
-test/unit/language/test_core.py::test_bin_op[1-uint64-uint16-%]
-test/unit/language/test_core.py::test_bin_op[1-uint64-uint32-%]
-test/unit/language/test_core.py::test_bin_op[1-uint64-uint64-%]
-test/unit/language/test_conversions.py::test_typeconvert_upcast[float8e4b8-float32]
-test/unit/language/test_conversions.py::test_typeconvert_upcast[float8e4b8-float16]
-test/unit/language/test_conversions.py::test_typeconvert_upcast[float8e5b16-float32]
-test/unit/language/test_conversions.py::test_typeconvert_upcast[float8e5b16-float16]
-test/unit/language/test_core.py::test_floordiv[1-uint8-uint8]
-test/unit/language/test_core.py::test_floordiv[1-uint8-uint16]
-test/unit/language/test_core.py::test_floordiv[1-uint8-uint32]
-test/unit/language/test_core.py::test_floordiv[1-uint8-uint64]
-test/unit/language/test_core.py::test_floordiv[1-uint16-uint8]
-test/unit/language/test_core.py::test_floordiv[1-uint16-uint16]
-test/unit/language/test_core.py::test_floordiv[1-uint16-uint32]
-test/unit/language/test_core.py::test_floordiv[1-uint16-uint64]
-test/unit/language/test_core.py::test_floordiv[1-uint32-uint8]
-test/unit/language/test_core.py::test_floordiv[1-uint32-uint16]
-test/unit/language/test_core.py::test_floordiv[1-uint32-uint32]
-test/unit/language/test_core.py::test_floordiv[1-uint32-uint64]
-test/unit/language/test_core.py::test_floordiv[1-uint64-uint8]
-test/unit/language/test_core.py::test_floordiv[1-uint64-uint16]
-test/unit/language/test_core.py::test_floordiv[1-uint64-uint32]
-test/unit/language/test_core.py::test_floordiv[1-uint64-uint64]
-test/unit/language/test_core.py::test_bin_op[1-uint8-uint8-/]
-test/unit/language/test_core.py::test_bin_op[1-uint8-uint16-/]
-test/unit/language/test_core.py::test_bin_op[1-uint8-uint32-/]
-test/unit/language/test_core.py::test_bin_op[1-uint8-uint64-/]
-test/unit/language/test_core.py::test_bin_op[1-uint16-uint8-/]
-test/unit/language/test_core.py::test_bin_op[1-uint16-uint16-/]
-test/unit/language/test_core.py::test_bin_op[1-uint16-uint32-/]
-test/unit/language/test_core.py::test_bin_op[1-uint16-uint64-/]
-test/unit/language/test_core.py::test_bin_op[1-uint32-uint8-/]
-test/unit/language/test_core.py::test_bin_op[1-uint32-uint16-/]
-test/unit/language/test_core.py::test_bin_op[1-uint32-uint32-/]
-test/unit/language/test_core.py::test_bin_op[1-uint32-uint64-/]
-test/unit/language/test_core.py::test_bin_op[1-uint64-uint8-/]
-test/unit/language/test_core.py::test_bin_op[1-uint64-uint16-/]
-test/unit/language/test_core.py::test_bin_op[1-uint64-uint32-/]
-test/unit/language/test_core.py::test_bin_op[1-uint64-uint64-/]
-test/unit/language/test_core.py::test_const[ternary-False-True]
-test/unit/language/test_core.py::test_const[ternary-False-False]
-test/unit/language/test_core.py::test_const[if-False-True]
-test/unit/language/test_core.py::test_const[if-False-False]
-test/unit/language/test_core.py::test_load_cache_modifier[.cv]
-test/unit/language/test_core.py::test_unroll_attr
diff --git a/scripts/skiplist/conda/subprocess.txt b/scripts/skiplist/conda/subprocess.txt
deleted file mode 100644
index 6f0ada5109..0000000000
--- a/scripts/skiplist/conda/subprocess.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-# https://github.com/intel/intel-xpu-backend-for-triton/issues/800
-test/unit/language/test_subprocess.py::test_print[device_print-float16]
-test/unit/language/test_subprocess.py::test_print[device_print-float32]
-test/unit/language/test_subprocess.py::test_print[device_print-float64]
-test/unit/language/test_subprocess.py::test_print[device_print_scalar-float16]
-test/unit/language/test_subprocess.py::test_print[device_print_scalar-float64]
-test/unit/language/test_subprocess.py::test_print[device_print_scalar-float32]
-# https://github.com/intel/intel-xpu-backend-for-triton/issues/1704
-test/unit/language/test_subprocess.py::test_print[device_print_uint-uint32]
diff --git a/scripts/skiplist/no-basekit/language.txt b/scripts/skiplist/no-basekit/language.txt
deleted file mode 100644
index f6da599e95..0000000000
--- a/scripts/skiplist/no-basekit/language.txt
+++ /dev/null
@@ -1,162 +0,0 @@
-test/unit/language/test_core.py::test_precise_math[1-tl.math.sqrt_rn(x)-tl.math.sqrt(x.to(tl.float64)).to(tl.float32)]
-test/unit/language/test_core.py::test_dot3d[1-1-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-1-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-1-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-1-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[1-1-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-1-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-1-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-1-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[1-16-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-16-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-16-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-16-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[1-16-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-16-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-16-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-16-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[1-2-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-2-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-2-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-2-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[1-2-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-2-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-2-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-2-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[1-4-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-4-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-4-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-4-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[1-4-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-4-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-4-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-4-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[1-8-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-8-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-8-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-8-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[1-8-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-8-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-8-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-8-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-1-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-1-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-1-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-1-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-1-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-1-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-1-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-1-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-16-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-16-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-16-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-16-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-16-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-16-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-16-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-16-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-2-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-2-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-2-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-2-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-2-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-2-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-2-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-2-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-4-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-4-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-4-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-4-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-4-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-4-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-4-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-4-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-8-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-8-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-8-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-8-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-8-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-8-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-8-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-8-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-1-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-1-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-1-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-1-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-1-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-1-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-1-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-1-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-16-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-16-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-16-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-16-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-16-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-16-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-16-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-16-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-2-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-2-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-2-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-2-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-2-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-2-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-2-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-2-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-4-128-128-64-64-64-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-4-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-4-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-4-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-4-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-4-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-4-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-4-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-4-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-8-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-8-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-8-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-8-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-8-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-8-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-8-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-8-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-1-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-1-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-1-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-1-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-1-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-1-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-1-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-1-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-16-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-16-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-16-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-16-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-16-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-16-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-16-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-16-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-2-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-2-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-2-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-2-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-2-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-2-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-2-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-2-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-4-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-4-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-4-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-4-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-4-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-4-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-4-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-4-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-8-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-8-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-8-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-8-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-8-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-8-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-8-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-8-64-64-64-32-32-int8-int8]
diff --git a/scripts/skiplist/no-basekit/regression.txt b/scripts/skiplist/no-basekit/regression.txt
deleted file mode 100644
index 43d097f565..0000000000
--- a/scripts/skiplist/no-basekit/regression.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-test/regression/test_functional_regressions.py::test_chained_matmul
-test/regression/test_functional_regressions.py::test_iv_dependent_matmul[pre_load]
-test/regression/test_functional_regressions.py::test_iv_dependent_matmul[post_load]
-test/regression/test_functional_regressions.py::test_iv_dependent_matmul[post_pre_mixed]
-test/regression/test_functional_regressions.py::test_iv_dependent_matmul[post_load_two_iters]
-test/regression/test_functional_regressions.py::test_iv_dependent_matmul[post_load_three_iters]
diff --git a/scripts/skiplist/no-basekit/subprocess.txt b/scripts/skiplist/no-basekit/subprocess.txt
deleted file mode 100644
index 6f0ada5109..0000000000
--- a/scripts/skiplist/no-basekit/subprocess.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-# https://github.com/intel/intel-xpu-backend-for-triton/issues/800
-test/unit/language/test_subprocess.py::test_print[device_print-float16]
-test/unit/language/test_subprocess.py::test_print[device_print-float32]
-test/unit/language/test_subprocess.py::test_print[device_print-float64]
-test/unit/language/test_subprocess.py::test_print[device_print_scalar-float16]
-test/unit/language/test_subprocess.py::test_print[device_print_scalar-float64]
-test/unit/language/test_subprocess.py::test_print[device_print_scalar-float32]
-# https://github.com/intel/intel-xpu-backend-for-triton/issues/1704
-test/unit/language/test_subprocess.py::test_print[device_print_uint-uint32]
diff --git a/scripts/skiplist/no-basekit/tutorials.txt b/scripts/skiplist/no-basekit/tutorials.txt
deleted file mode 100644
index daf952d619..0000000000
--- a/scripts/skiplist/no-basekit/tutorials.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-03-matrix-multiplication
-06-fused-attention
-10-experimental-block-pointer
-10i-experimental-block-pointer

From 3035a09305616e4d364d0141eab12c202c9e1da3 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Tue, 8 Oct 2024 15:28:01 +0200
Subject: [PATCH 027/105] Minor changes, trying to debug

---
 .github/workflows/build-test-reusable.yml   |  9 ++++-
 .github/workflows/conda-build-test.yml      | 43 +--------------------
 .github/workflows/no-basekit-build-test.yml | 41 --------------------
 3 files changed, 9 insertions(+), 84 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 66653a7c4e..5069cabbd9 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -212,12 +212,19 @@ jobs:
           grep AlbertForMaskedLM $WORKSPACE/inductor_log/*/*/*.csv | grep -q ,pass,
 
       - name: Save pip cache
-        if: ${{ steps.pip-cache.outputs.status == 'miss' }}
+        if: (inputs.env_manager == 'base') && (steps.pip-cache.outputs.status == 'miss')
         uses: ./.github/actions/save
         with:
           path: ${{ steps.pip-cache.outputs.path }}
           dest: ${{ steps.pip-cache.outputs.dest }}
 
+      - name: Save conda cache
+        if: (inputs.env_manager != 'base') && (steps.conda-cache.outputs.status == 'miss')
+        uses: ./.github/actions/save
+        with:
+          path: ${{ steps.conda-cache.outputs.path }}
+          dest: ${{ steps.conda-cache.outputs.dest }}
+
       - name: Pass rate
         run: |
           source ./scripts/capture-hw-details.sh
diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml
index a0d7bb7ca2..2e6477038b 100644
--- a/.github/workflows/conda-build-test.yml
+++ b/.github/workflows/conda-build-test.yml
@@ -1,4 +1,4 @@
-name: Conda test
+name: Conda Test
 run-name: ${{ inputs.run_name }}
 
 on:
@@ -49,47 +49,6 @@ on:
 permissions: read-all
 
 jobs:
-  pre-commit:
-    name: Pre-commit checks
-    runs-on: Linux
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Load pip cache
-        id: pip-cache
-        uses: ./.github/actions/load
-        env:
-          # Increase this value to reset cache
-          CACHE_NUMBER: 2
-        with:
-          path: $HOME/.cache/pip
-          key: pip-3.10-${{ hashFiles('.pre-commit-config.yaml') }}-${{ env.CACHE_NUMBER }}
-
-      - name: Install Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Run pre-commit checks
-        run: |
-          set -x
-          pip install --upgrade pre-commit
-
-          # TODO: ignore the first yapf failure until https://github.com/google/yapf/issues/1164 is fixed
-          python3 -m pre_commit run --all-files --verbose yapf &> /dev/null || true
-          # If first run of yapf worked and made changes reset the tree to the original state
-          git reset --hard
-
-          python3 -m pre_commit run --show-diff-on-failure --color=always --all-files --verbose
-
-      - name: Save pip cache
-        if: ${{ steps.pip-cache.outputs.status == 'miss' }}
-        uses: ./.github/actions/save
-        with:
-          path: ${{ steps.pip-cache.outputs.path }}
-          dest: ${{ steps.pip-cache.outputs.dest }}
-
   prepare:
     name: Prepare
     runs-on: Linux
diff --git a/.github/workflows/no-basekit-build-test.yml b/.github/workflows/no-basekit-build-test.yml
index 1bf8f94577..314ebc88d5 100644
--- a/.github/workflows/no-basekit-build-test.yml
+++ b/.github/workflows/no-basekit-build-test.yml
@@ -49,47 +49,6 @@ on:
 permissions: read-all
 
 jobs:
-  pre-commit:
-    name: Pre-commit checks
-    runs-on: Linux
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Load pip cache
-        id: pip-cache
-        uses: ./.github/actions/load
-        env:
-          # Increase this value to reset cache
-          CACHE_NUMBER: 2
-        with:
-          path: $HOME/.cache/pip
-          key: pip-3.10-${{ hashFiles('.pre-commit-config.yaml') }}-${{ env.CACHE_NUMBER }}
-
-      - name: Install Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Run pre-commit checks
-        run: |
-          set -x
-          pip install --upgrade pre-commit
-
-          # TODO: ignore the first yapf failure until https://github.com/google/yapf/issues/1164 is fixed
-          python3 -m pre_commit run --all-files --verbose yapf &> /dev/null || true
-          # If first run of yapf worked and made changes reset the tree to the original state
-          git reset --hard
-
-          python3 -m pre_commit run --show-diff-on-failure --color=always --all-files --verbose
-
-      - name: Save pip cache
-        if: ${{ steps.pip-cache.outputs.status == 'miss' }}
-        uses: ./.github/actions/save
-        with:
-          path: ${{ steps.pip-cache.outputs.path }}
-          dest: ${{ steps.pip-cache.outputs.dest }}
-
   prepare:
     name: Prepare
     runs-on: Linux

From 79c1bdb6d2744f5a40b71c9dc15cf6ee5101472a Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Tue, 8 Oct 2024 15:37:15 +0200
Subject: [PATCH 028/105] Update conda cache id

---
 .github/workflows/build-test-reusable.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 5069cabbd9..a9e53dce31 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -89,7 +89,7 @@ jobs:
         id: conda-cache
         uses: ./.github/actions/load
         env:
-          CACHE_NUMBER: 5
+          CACHE_NUMBER: 6
         with:
           path: $HOME/miniforge3/envs/triton
           key: conda-${{ inputs.env_manager }}-py${{ matrix.python }}-${{ hashFiles('scripts/triton.yml', 'python/pyproject.toml', 'python/setup.py') }}-${{ env.CACHE_NUMBER }}

From 871975d971ed3941cb4de468f93c0830b09570bd Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Tue, 8 Oct 2024 15:42:28 +0200
Subject: [PATCH 029/105] Trying to workaround a cache

---
 .github/workflows/build-test-reusable.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index a9e53dce31..76385523e8 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -84,6 +84,10 @@ jobs:
           ${{ toJSON(inputs) }}
           EOF
 
+      - name: FIXME Remove the env
+        run: |
+          rm -rf $HOME/miniforge3/envs/triton
+
       - name: Load conda cache
         if: inputs.env_manager != 'base'
         id: conda-cache

From d5d634bb0e3d4e629ed4b674308586b602309fcf Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Tue, 8 Oct 2024 16:06:03 +0200
Subject: [PATCH 030/105] Fix undefined var

---
 .github/workflows/build-test-reusable.yml | 1 +
 scripts/pytest-utils.sh                   | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 76385523e8..19ff4a7718 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -86,6 +86,7 @@ jobs:
 
       - name: FIXME Remove the env
         run: |
+          ls -l $HOME/miniforge3/envs/triton || true
           rm -rf $HOME/miniforge3/envs/triton
 
       - name: Load conda cache
diff --git a/scripts/pytest-utils.sh b/scripts/pytest-utils.sh
index 80e0fd3712..6f75316e16 100644
--- a/scripts/pytest-utils.sh
+++ b/scripts/pytest-utils.sh
@@ -92,8 +92,8 @@ run_tutorial_test() {
 capture_runtime_env() {
     mkdir -p "$TRITON_TEST_REPORTS_DIR"
 
-    echo "$CMPLR_ROOT" > $TRITON_TEST_REPORTS_DIR/cmplr_version.txt
-    echo "$MKLROOT" > $TRITON_TEST_REPORTS_DIR/mkl_version.txt
+    echo "$CMPLR_ROOT" > $TRITON_TEST_REPORTS_DIR/cmplr_version.txt || true
+    echo "$MKLROOT" > $TRITON_TEST_REPORTS_DIR/mkl_version.txt || true
 
     # Exit script execution as long as one of those components is not found.
     local TRITON_COMMIT=""

From 2261b14dcb996adfb2569a6852a094e1382d08f3 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Tue, 8 Oct 2024 16:30:32 +0200
Subject: [PATCH 031/105] Fix undefined var, r1

---
 scripts/pytest-utils.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/pytest-utils.sh b/scripts/pytest-utils.sh
index 6f75316e16..36404b492c 100644
--- a/scripts/pytest-utils.sh
+++ b/scripts/pytest-utils.sh
@@ -92,8 +92,10 @@ run_tutorial_test() {
 capture_runtime_env() {
     mkdir -p "$TRITON_TEST_REPORTS_DIR"
 
-    echo "$CMPLR_ROOT" > $TRITON_TEST_REPORTS_DIR/cmplr_version.txt || true
-    echo "$MKLROOT" > $TRITON_TEST_REPORTS_DIR/mkl_version.txt || true
+    set +u
+    echo "$CMPLR_ROOT" > $TRITON_TEST_REPORTS_DIR/cmplr_version.txt
+    echo "$MKLROOT" > $TRITON_TEST_REPORTS_DIR/mkl_version.txt
+    set -u
 
     # Exit script execution as long as one of those components is not found.
     local TRITON_COMMIT=""

From 12724749f02ab2cd6cfb3193ea6e8b262f05fb94 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Tue, 8 Oct 2024 17:26:09 +0200
Subject: [PATCH 032/105] Try a skiplist

---
 .github/workflows/build-test-reusable.yml |  17 ++--
 scripts/run_util.sh                       |   2 +
 scripts/skiplist/conda/language.txt       | 102 ++++++++++++++++++++++
 3 files changed, 110 insertions(+), 11 deletions(-)
 create mode 100644 scripts/skiplist/conda/language.txt

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 19ff4a7718..c75e4101bc 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -78,17 +78,6 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Print inputs
-        run: |
-          cat <<EOF
-          ${{ toJSON(inputs) }}
-          EOF
-
-      - name: FIXME Remove the env
-        run: |
-          ls -l $HOME/miniforge3/envs/triton || true
-          rm -rf $HOME/miniforge3/envs/triton
-
       - name: Load conda cache
         if: inputs.env_manager != 'base'
         id: conda-cache
@@ -99,6 +88,12 @@ jobs:
           path: $HOME/miniforge3/envs/triton
           key: conda-${{ inputs.env_manager }}-py${{ matrix.python }}-${{ hashFiles('scripts/triton.yml', 'python/pyproject.toml', 'python/setup.py') }}-${{ env.CACHE_NUMBER }}
 
+      - name: Print inputs
+        run: |
+          cat <<EOF
+          ${{ toJSON(inputs) }}
+          EOF
+
       - name: Load pip cache
         if: inputs.env_manager == 'base'
         id: pip-cache
diff --git a/scripts/run_util.sh b/scripts/run_util.sh
index 8b0a353821..719477dcbf 100644
--- a/scripts/run_util.sh
+++ b/scripts/run_util.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 
 print_conda_info() {
+    test ! -f first_run || return 0
+    touch first_run
     conda info
     conda list -n triton
 }
diff --git a/scripts/skiplist/conda/language.txt b/scripts/skiplist/conda/language.txt
new file mode 100644
index 0000000000..3c7ee3829c
--- /dev/null
+++ b/scripts/skiplist/conda/language.txt
@@ -0,0 +1,102 @@
+test/unit/language/test_core.py::test_precise_math[1-tl.math.sqrt_rn(x)-tl.math.sqrt(x.to(tl.float64)).to(tl.float32)]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint8-%]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint16-%]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint32-%]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint64-%]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint8-%]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint16-%]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint32-%]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint64-%]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint8-%]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint16-%]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint32-%]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint64-%]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint8-%]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint16-%]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint32-%]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint64-%]
+test/unit/language/test_compile_errors.py::test_where_warning
+test/unit/language/test_compile_errors.py::test_fp8_support[dtype1]
+test/unit/language/test_compile_errors.py::test_fp8_support[dtype3]
+test/unit/language/test_conversions.py::test_typeconvert_upcast[float8e4b8-float32]
+test/unit/language/test_conversions.py::test_typeconvert_upcast[float8e4b8-float16]
+test/unit/language/test_conversions.py::test_typeconvert_upcast[float8e5b16-float32]
+test/unit/language/test_conversions.py::test_typeconvert_upcast[float8e5b16-float16]
+test/unit/language/test_core.py::test_floordiv[1-uint8-uint8]
+test/unit/language/test_core.py::test_floordiv[1-uint8-uint16]
+test/unit/language/test_core.py::test_floordiv[1-uint8-uint32]
+test/unit/language/test_core.py::test_floordiv[1-uint8-uint64]
+test/unit/language/test_core.py::test_floordiv[1-uint16-uint8]
+test/unit/language/test_core.py::test_floordiv[1-uint16-uint16]
+test/unit/language/test_core.py::test_floordiv[1-uint16-uint32]
+test/unit/language/test_core.py::test_floordiv[1-uint16-uint64]
+test/unit/language/test_core.py::test_floordiv[1-uint32-uint8]
+test/unit/language/test_core.py::test_floordiv[1-uint32-uint16]
+test/unit/language/test_core.py::test_floordiv[1-uint32-uint32]
+test/unit/language/test_core.py::test_floordiv[1-uint32-uint64]
+test/unit/language/test_core.py::test_floordiv[1-uint64-uint8]
+test/unit/language/test_core.py::test_floordiv[1-uint64-uint16]
+test/unit/language/test_core.py::test_floordiv[1-uint64-uint32]
+test/unit/language/test_core.py::test_floordiv[1-uint64-uint64]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint8-/]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint16-/]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint32-/]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint64-/]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint8-/]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint16-/]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint32-/]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint64-/]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint8-/]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint16-/]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint32-/]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint64-/]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint8-/]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint16-/]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint32-/]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint64-/]
+test/unit/language/test_core.py::test_dot3d[1-4-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_load_cache_modifier[.cv]
+test/unit/language/test_core.py::test_const[ternary-False-True]
+test/unit/language/test_core.py::test_const[ternary-False-False]
+test/unit/language/test_core.py::test_const[if-False-True]
+test/unit/language/test_core.py::test_const[if-False-False]
+test/unit/language/test_core.py::test_unroll_attr
+test/unit/language/test_core.py::test_dot3d[1-4-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[1-8-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[2-2-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[2-4-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[2-1-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[2-8-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[1-16-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[4-4-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[2-16-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[4-1-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[4-2-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[4-8-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[4-16-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[8-1-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[8-2-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[8-4-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[8-8-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[8-16-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[4-16-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[1-16-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[2-8-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[1-4-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[1-4-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[1-8-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[2-1-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[2-4-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[2-2-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[2-16-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[2-16-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[4-4-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[4-1-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[4-2-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[4-8-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[4-8-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[8-1-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[8-2-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[8-4-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[8-4-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[1-1-64-64-64-32-32-int8-int8]

From 403bd46cca5fe4ceec793a78db5a71c7e00794fa Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Wed, 9 Oct 2024 14:19:07 +0200
Subject: [PATCH 033/105] Update skiplists

---
 .github/workflows/conda-build-test.yml |   3 +-
 scripts/skiplist/conda/language.txt    | 318 +++++++++++++++++++++----
 scripts/skiplist/conda/subprocess.txt  |   6 +
 3 files changed, 285 insertions(+), 42 deletions(-)
 create mode 100644 scripts/skiplist/conda/subprocess.txt

diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml
index 2e6477038b..0c0223645e 100644
--- a/.github/workflows/conda-build-test.yml
+++ b/.github/workflows/conda-build-test.yml
@@ -88,8 +88,7 @@ jobs:
       pytorch_mode: ${{ inputs.pytorch_mode || 'source' }}
       python_version: ${{ matrix.python }}
       upload_test_reports: ${{ inputs.upload_test_reports || false }}
-      ignore_errors: true
-      #FIXME${{ inputs.ignore_errors || false }}
+      ignore_errors: ${{ inputs.ignore_errors || false }}
       skip_list: conda
       run_name: ${{ inputs.run_name }}
       enable_unskip: ${{ inputs.enable_unskip || false }}
diff --git a/scripts/skiplist/conda/language.txt b/scripts/skiplist/conda/language.txt
index 3c7ee3829c..4c618d732f 100644
--- a/scripts/skiplist/conda/language.txt
+++ b/scripts/skiplist/conda/language.txt
@@ -1,3 +1,281 @@
+# LTS failures
+test/unit/language/test_core.py::test_dot[1-64-128-128-4-True-True-none-tf32-int8-int8-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-4-True-True-none-tf32-int8-int8-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-4-False-True-none-tf32-int8-int8-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-4-False-True-none-tf32-int8-int8-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-True-True-none-tf32-int8-int8-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-True-True-none-tf32-int8-int8-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-True-True-none-tf32-int8-int8-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-True-True-none-tf32-int8-int8-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-True-True-none-tf32-float16-float16-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-True-True-none-tf32-float16-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-True-True-none-tf32-float16-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-True-False-none-tf32-int8-int8-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-True-False-none-tf32-int8-int8-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-True-True-none-tf32-float16-float16-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-True-False-none-tf32-float16-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-True-False-none-tf32-float16-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-True-True-none-tf32-float16-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-False-True-none-tf32-int8-int8-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-False-True-none-tf32-int8-int8-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-True-True-none-tf32-float16-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-True-True-none-tf32-float32-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-False-True-none-tf32-float16-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-False-True-none-tf32-float16-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-True-True-none-tf32-float32-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-False-False-none-tf32-int8-int8-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-True-False-none-tf32-int8-int8-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-False-False-none-tf32-int8-int8-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-True-False-none-tf32-int8-int8-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-False-False-none-tf32-float16-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-True-False-none-tf32-float16-float16-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-False-False-none-tf32-float16-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-True-False-none-tf32-float16-float16-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-4-True-True-none-tf32-float16-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-4-True-True-none-tf32-float16-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-4-True-False-none-tf32-int8-int8-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-True-False-none-tf32-float16-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-4-True-False-none-tf32-int8-int8-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-True-False-none-tf32-float16-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-4-True-False-none-tf32-float16-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-4-True-False-none-tf32-float16-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-True-False-none-tf32-float32-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-True-False-none-tf32-float32-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-4-False-True-none-tf32-float16-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-False-True-none-tf32-int8-int8-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-4-False-True-none-tf32-float16-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-4-False-False-none-tf32-int8-int8-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-4-False-False-none-tf32-int8-int8-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-False-True-none-tf32-int8-int8-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-4-False-False-none-tf32-float16-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-4-False-False-none-tf32-float16-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-False-True-none-tf32-float16-float16-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-False-True-none-tf32-float16-float16-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-False-True-none-tf32-float16-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-False-True-none-tf32-float16-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-False-True-none-tf32-float32-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-False-True-none-tf32-float32-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-False-False-none-tf32-int8-int8-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-False-False-none-tf32-int8-int8-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-False-False-none-tf32-float16-float16-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-False-False-none-tf32-float16-float16-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-False-False-none-tf32-float16-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-False-False-none-tf32-float16-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-False-False-none-tf32-float32-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-2-False-False-none-tf32-float32-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-True-True-none-tf32-int8-int8-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-True-True-none-tf32-int8-int8-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-True-True-none-tf32-float16-float16-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-True-True-none-tf32-float16-float16-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-True-True-none-tf32-float16-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-True-True-none-tf32-float16-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-True-True-none-tf32-float32-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-True-True-none-tf32-float32-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-True-False-none-tf32-int8-int8-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-True-False-none-tf32-int8-int8-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-True-False-none-tf32-float16-float16-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-True-False-none-tf32-float16-float16-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-True-False-none-tf32-float16-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-True-False-none-tf32-float16-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-True-False-none-tf32-float32-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-True-False-none-tf32-float32-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-False-True-none-tf32-int8-int8-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-False-True-none-tf32-int8-int8-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-False-True-none-tf32-float16-float16-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-False-True-none-tf32-float16-float16-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-False-True-none-tf32-float16-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-False-True-none-tf32-float16-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-False-True-none-tf32-float32-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-False-True-none-tf32-float32-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-False-False-none-tf32-int8-int8-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-False-False-none-tf32-int8-int8-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-False-False-none-tf32-float16-float16-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-False-False-none-tf32-float16-float16-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-False-False-none-tf32-float16-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-False-False-none-tf32-float16-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-False-False-none-tf32-float32-float32-1_0]
+test/unit/language/test_core.py::test_dot[1-64-128-128-2-False-False-none-tf32-float32-float32-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-False-False-chain-dot-ieee-float8e5-float32-1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-False-False-chain-dot-ieee-float8e4nv-float32-1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-False-True-none-tf32-float16-float16-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-False-True-none-tf32-float16-float16-1_1]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-False-False-none-tf32-float16-float16-1_0]
+test/unit/language/test_core.py::test_dot[1-128-128-64-4-False-False-none-tf32-float16-float16-1_1]
+test/unit/language/test_core.py::test_dot_max_num_imprecise_acc[0-float8e5-128-256-128-128-256-256]
+test/unit/language/test_core.py::test_dot_max_num_imprecise_acc[32-float8e5-128-256-128-128-256-256]
+test/unit/language/test_core.py::test_dot_max_num_imprecise_acc[128-float8e4nv-128-256-128-128-256-256]
+test/unit/language/test_core.py::test_dot_max_num_imprecise_acc[0-float8e4nv-128-256-128-128-256-256]
+test/unit/language/test_core.py::test_dot_max_num_imprecise_acc[32-float8e4nv-128-256-128-128-256-256]
+test/unit/language/test_core.py::test_dot_max_num_imprecise_acc[128-float8e4b15-128-256-128-128-256-256]
+test/unit/language/test_core.py::test_dot_max_num_imprecise_acc[0-float8e4b15-128-256-128-128-256-256]
+test/unit/language/test_core.py::test_dot_max_num_imprecise_acc[32-float8e4b15-128-256-128-128-256-256]
+test/unit/language/test_core.py::test_dot_max_num_imprecise_acc[64-float8e5-128-256-128-128-256-256]
+test/unit/language/test_core.py::test_dot_max_num_imprecise_acc[64-float8e4nv-128-256-128-128-256-256]
+test/unit/language/test_core.py::test_dot_max_num_imprecise_acc[64-float8e4b15-128-256-128-128-256-256]
+test/unit/language/test_core.py::test_dot_max_num_imprecise_acc[128-float8e5-128-256-128-128-256-256]
+# https://github.com/intel/intel-xpu-backend-for-triton/issues/1434
+test/unit/language/test_core.py::test_precise_math[1-tl.math.sqrt_rn(x)-tl.math.sqrt(x.to(tl.float64)).to(tl.float32)]
+test/unit/language/test_core.py::test_dot3d[1-1-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[1-1-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[1-1-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[1-1-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[1-1-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[1-1-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[1-1-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[1-1-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[1-16-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[1-16-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[1-16-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[1-16-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[1-16-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[1-16-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[1-16-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[1-16-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[1-2-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[1-2-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[1-2-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[1-2-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[1-2-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[1-2-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[1-2-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[1-2-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[1-4-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[1-4-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[1-4-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[1-4-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[1-4-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[1-4-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[1-4-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[1-4-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[1-8-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[1-8-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[1-8-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[1-8-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[1-8-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[1-8-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[1-8-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[1-8-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[2-1-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[2-1-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[2-1-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[2-1-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[2-1-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[2-1-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[2-1-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[2-1-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[2-16-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[2-16-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[2-16-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[2-16-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[2-16-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[2-16-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[2-16-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[2-16-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[2-2-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[2-2-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[2-2-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[2-2-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[2-2-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[2-2-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[2-2-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[2-2-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[2-4-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[2-4-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[2-4-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[2-4-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[2-4-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[2-4-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[2-4-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[2-4-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[2-8-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[2-8-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[2-8-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[2-8-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[2-8-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[2-8-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[2-8-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[2-8-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[4-1-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[4-1-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[4-1-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[4-1-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[4-1-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[4-1-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[4-1-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[4-1-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[4-16-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[4-16-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[4-16-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[4-16-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[4-16-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[4-16-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[4-16-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[4-16-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[4-2-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[4-2-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[4-2-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[4-2-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[4-2-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[4-2-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[4-2-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[4-2-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[4-4-128-128-64-64-64-float16-float16]
+test/unit/language/test_core.py::test_dot3d[4-4-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[4-4-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[4-4-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[4-4-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[4-4-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[4-4-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[4-4-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[4-4-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[4-8-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[4-8-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[4-8-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[4-8-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[4-8-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[4-8-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[4-8-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[4-8-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[8-1-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[8-1-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[8-1-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[8-1-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[8-1-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[8-1-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[8-1-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[8-1-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[8-16-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[8-16-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[8-16-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[8-16-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[8-16-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[8-16-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[8-16-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[8-16-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[8-2-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[8-2-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[8-2-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[8-2-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[8-2-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[8-2-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[8-2-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[8-2-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[8-4-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[8-4-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[8-4-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[8-4-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[8-4-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[8-4-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[8-4-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[8-4-64-64-64-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[8-8-32-32-32-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[8-8-32-32-32-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[8-8-32-32-32-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[8-8-32-32-32-32-32-int8-int8]
+test/unit/language/test_core.py::test_dot3d[8-8-64-64-64-32-32-float16-float16]
+test/unit/language/test_core.py::test_dot3d[8-8-64-64-64-32-32-float16-float32]
+test/unit/language/test_core.py::test_dot3d[8-8-64-64-64-32-32-float32-float32]
+test/unit/language/test_core.py::test_dot3d[8-8-64-64-64-32-32-int8-int8]
 test/unit/language/test_core.py::test_precise_math[1-tl.math.sqrt_rn(x)-tl.math.sqrt(x.to(tl.float64)).to(tl.float32)]
 test/unit/language/test_core.py::test_bin_op[1-uint8-uint8-%]
 test/unit/language/test_core.py::test_bin_op[1-uint8-uint16-%]
@@ -54,49 +332,9 @@ test/unit/language/test_core.py::test_bin_op[1-uint64-uint8-/]
 test/unit/language/test_core.py::test_bin_op[1-uint64-uint16-/]
 test/unit/language/test_core.py::test_bin_op[1-uint64-uint32-/]
 test/unit/language/test_core.py::test_bin_op[1-uint64-uint64-/]
-test/unit/language/test_core.py::test_dot3d[1-4-64-64-64-32-32-float16-float16]
 test/unit/language/test_core.py::test_load_cache_modifier[.cv]
 test/unit/language/test_core.py::test_const[ternary-False-True]
 test/unit/language/test_core.py::test_const[ternary-False-False]
 test/unit/language/test_core.py::test_const[if-False-True]
 test/unit/language/test_core.py::test_const[if-False-False]
 test/unit/language/test_core.py::test_unroll_attr
-test/unit/language/test_core.py::test_dot3d[1-4-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[1-8-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-2-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-4-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-1-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-8-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-16-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-4-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-16-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-1-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-2-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-8-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[4-16-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[8-1-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-2-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-4-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-8-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-16-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-16-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[1-16-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-8-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[1-4-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-4-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[1-8-32-32-32-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[2-1-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[2-4-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-2-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[2-16-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[2-16-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[4-4-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-1-32-32-32-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-2-32-32-32-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[4-8-64-64-64-32-32-float16-float32]
-test/unit/language/test_core.py::test_dot3d[4-8-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-1-64-64-64-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[8-2-64-64-64-32-32-float16-float16]
-test/unit/language/test_core.py::test_dot3d[8-4-64-64-64-32-32-int8-int8]
-test/unit/language/test_core.py::test_dot3d[8-4-32-32-32-32-32-float32-float32]
-test/unit/language/test_core.py::test_dot3d[1-1-64-64-64-32-32-int8-int8]
diff --git a/scripts/skiplist/conda/subprocess.txt b/scripts/skiplist/conda/subprocess.txt
new file mode 100644
index 0000000000..8895b4638b
--- /dev/null
+++ b/scripts/skiplist/conda/subprocess.txt
@@ -0,0 +1,6 @@
+test/unit/language/test_subprocess.py::test_print[device_print_scalar-float32]
+test/unit/language/test_subprocess.py::test_print[device_print-float32]
+test/unit/language/test_subprocess.py::test_print[device_print_scalar-float16]
+test/unit/language/test_subprocess.py::test_print[device_print-float16]
+test/unit/language/test_subprocess.py::test_print[device_print_scalar-float64]
+test/unit/language/test_subprocess.py::test_print[device_print-float64]

From d0b71e6ba6f6832f92716382cfa12a9efd6575cc Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Wed, 9 Oct 2024 14:19:44 +0200
Subject: [PATCH 034/105] Update skiplists, r1

---
 scripts/skiplist/conda/interpreter.txt | 61 ++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 scripts/skiplist/conda/interpreter.txt

diff --git a/scripts/skiplist/conda/interpreter.txt b/scripts/skiplist/conda/interpreter.txt
new file mode 100644
index 0000000000..b1f075cc6a
--- /dev/null
+++ b/scripts/skiplist/conda/interpreter.txt
@@ -0,0 +1,61 @@
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint32-%]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint64-/]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint64-%]
+test/unit/language/test_core.py::test_floordiv[1-uint8-uint8]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint8-%]
+test/unit/language/test_core.py::test_floordiv[1-uint8-uint16]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint16-%]
+test/unit/language/test_core.py::test_floordiv[1-uint8-uint32]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint32-%]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint64-%]
+test/unit/language/test_core.py::test_floordiv[1-uint8-uint64]
+test/unit/language/test_core.py::test_floordiv[1-uint16-uint8]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint8-%]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint16-%]
+test/unit/language/test_core.py::test_floordiv[1-uint16-uint16]
+test/unit/language/test_core.py::test_floordiv[1-uint16-uint32]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint32-%]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint8-/]
+test/unit/language/test_core.py::test_floordiv[1-uint16-uint64]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint64-%]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint16-/]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint32-/]
+test/unit/language/test_core.py::test_floordiv[1-uint32-uint8]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint8-%]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint64-/]
+test/unit/language/test_core.py::test_floordiv[1-uint32-uint16]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint16-%]
+test/unit/language/test_core.py::test_floordiv[1-uint32-uint32]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint8-/]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint32-%]
+test/unit/language/test_core.py::test_floordiv[1-uint32-uint64]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint16-/]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint64-%]
+test/unit/language/test_core.py::test_floordiv[1-uint64-uint8]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint32-/]
+test/unit/language/test_core.py::test_bin_op[1-uint16-uint64-/]
+test/unit/language/test_core.py::test_floordiv[1-uint64-uint16]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint8-%]
+test/unit/language/test_core.py::test_floordiv[1-uint64-uint32]
+test/unit/language/test_core.py::test_reduce1d[4-1-1-sum-int8-32]
+test/unit/language/test_core.py::test_bin_op[1-uint8-uint16-%]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint8-/]
+test/unit/language/test_core.py::test_floordiv[1-uint64-uint64]
+test/unit/language/test_core.py::test_reduce1d[4-1-1-sum-int8-64]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint16-/]
+test/unit/language/test_core.py::test_reduce1d[4-1-1-sum-int8-128]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint32-/]
+test/unit/language/test_core.py::test_reduce1d[4-1-1-sum-int8-512]
+test/unit/language/test_core.py::test_bin_op[1-uint32-uint64-/]
+test/unit/language/test_core.py::test_reduce1d[4-1-1-sum-int16-32]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint8-/]
+test/unit/language/test_core.py::test_reduce1d[4-1-1-sum-int16-64]
+test/unit/language/test_core.py::test_reduce1d[4-1-1-sum-int16-128]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint16-/]
+test/unit/language/test_core.py::test_bin_op[1-uint64-uint32-/]
+test/unit/language/test_core.py::test_reduce1d[4-1-1-sum-int16-512]
+test/unit/language/test_core.py::test_reduce1d[4-1-1-sum-int32-32]
+test/unit/language/test_core.py::test_reduce1d[4-1-1-sum-int32-64]
+test/unit/language/test_core.py::test_reduce1d[4-1-1-sum-int32-128]
+test/unit/language/test_core.py::test_reduce1d[4-1-1-sum-int32-512]
+test/unit/language/test_core.py::test_load_cache_modifier[.cv]

From 6a0b73b60b6ea5347d7dc61adbe8afc1b23b27ba Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Wed, 9 Oct 2024 16:24:26 +0200
Subject: [PATCH 035/105] Update skiplists, r2

---
 scripts/skiplist/conda/runtime.txt   | 1 +
 scripts/skiplist/conda/tutorials.txt | 4 ++++
 2 files changed, 5 insertions(+)
 create mode 100644 scripts/skiplist/conda/runtime.txt
 create mode 100644 scripts/skiplist/conda/tutorials.txt

diff --git a/scripts/skiplist/conda/runtime.txt b/scripts/skiplist/conda/runtime.txt
new file mode 100644
index 0000000000..f7a443f7d6
--- /dev/null
+++ b/scripts/skiplist/conda/runtime.txt
@@ -0,0 +1 @@
+test/unit/runtime/test_bindings.py::test_module_walk
diff --git a/scripts/skiplist/conda/tutorials.txt b/scripts/skiplist/conda/tutorials.txt
new file mode 100644
index 0000000000..daf952d619
--- /dev/null
+++ b/scripts/skiplist/conda/tutorials.txt
@@ -0,0 +1,4 @@
+03-matrix-multiplication
+06-fused-attention
+10-experimental-block-pointer
+10i-experimental-block-pointer

From a2a78d70b2ee1da793122f9452dfc2cd3389714a Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Wed, 9 Oct 2024 16:49:04 +0200
Subject: [PATCH 036/105] Add debug for instrumentation tests

---
 scripts/test-triton.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/test-triton.sh b/scripts/test-triton.sh
index a372543482..f6cb08b434 100755
--- a/scripts/test-triton.sh
+++ b/scripts/test-triton.sh
@@ -298,6 +298,9 @@ run_instrumentation_tests() {
     return
   fi
 
+  #FIXME
+  find $TRITON_PROJ
+
   SHARED_LIB_DIR=$(ls -1d $TRITON_PROJ/python/build/*lib*/triton/_C) || err "Could not find $TRITON_PROJ/python/build/*lib*/triton/_C, build Triton first"
 
   cd $TRITON_PROJ/python/test/unit

From d9d6b5b1bcae3b4541c2b114376a3eeb41ce87de Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Wed, 9 Oct 2024 17:20:02 +0200
Subject: [PATCH 037/105] Fix libGPUHello.so location

---
 scripts/test-triton.sh | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/scripts/test-triton.sh b/scripts/test-triton.sh
index f6cb08b434..8df7009dcd 100755
--- a/scripts/test-triton.sh
+++ b/scripts/test-triton.sh
@@ -298,10 +298,7 @@ run_instrumentation_tests() {
     return
   fi
 
-  #FIXME
-  find $TRITON_PROJ
-
-  SHARED_LIB_DIR=$(ls -1d $TRITON_PROJ/python/build/*lib*/triton/_C) || err "Could not find $TRITON_PROJ/python/build/*lib*/triton/_C, build Triton first"
+  SHARED_LIB_DIR=$(ls -1d $TRITON_PROJ/python/build/*lib*/triton/_C) || SHARED_LIB_DIR=$(ls -1d $TRITON_PROJ/python/triton/_C) || err "Could not find $TRITON_PROJ/python/build/*lib*/triton/_C, build Triton first"
 
   cd $TRITON_PROJ/python/test/unit
 

From 210eda2ff2335ac7418a7dba958b927701608f9a Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Wed, 9 Oct 2024 17:46:26 +0200
Subject: [PATCH 038/105] Skip e2e

---
 .github/workflows/build-test-reusable.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index c75e4101bc..103e691ef3 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -190,16 +190,17 @@ jobs:
           echo "TRANSFORMERS_VERSION=$TRANSFORMERS_VERSION" | tee -a $GITHUB_ENV
 
       - name: Install transformers
-        if: ${{ inputs.python_version != '3.12' }}
+        if: inputs.env_manager == base && inputs.python_version != '3.12'
         uses: ./.github/actions/install-dependency
         with:
           package: transformers
           repository: huggingface/transformers
           ref: ${{ env.TRANSFORMERS_VERSION }}
           try-tag-prefix: v
+          env_manager: ${{ inputs.env-manager }}
 
       - name: Run E2E test
-        if: ${{ inputs.python_version != '3.12' }}
+        if: inputs.env_manager == base && nputs.python_version != '3.12'
         run: |
           # Set WORKSPACE for inductor_xpu_test.sh to make sure it creates "inductor_log" outside of pytorch cloned directory
           export WORKSPACE=$GITHUB_WORKSPACE

From 1fc39e201a3c6e097f9d82429179a1217d220195 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Wed, 9 Oct 2024 17:47:56 +0200
Subject: [PATCH 039/105] Skip e2e, r1

---
 .github/workflows/build-test-reusable.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 103e691ef3..92563b9bd9 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -190,7 +190,7 @@ jobs:
           echo "TRANSFORMERS_VERSION=$TRANSFORMERS_VERSION" | tee -a $GITHUB_ENV
 
       - name: Install transformers
-        if: inputs.env_manager == base && inputs.python_version != '3.12'
+        if: inputs.env_manager == 'base' && inputs.python_version != '3.12'
         uses: ./.github/actions/install-dependency
         with:
           package: transformers
@@ -200,7 +200,7 @@ jobs:
           env_manager: ${{ inputs.env-manager }}
 
       - name: Run E2E test
-        if: inputs.env_manager == base && nputs.python_version != '3.12'
+        if: inputs.env_manager == 'base' && nputs.python_version != '3.12'
         run: |
           # Set WORKSPACE for inductor_xpu_test.sh to make sure it creates "inductor_log" outside of pytorch cloned directory
           export WORKSPACE=$GITHUB_WORKSPACE

From 2014db6e7b6e66343e6e3820c75864e2dbd9d517 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Wed, 9 Oct 2024 17:48:31 +0200
Subject: [PATCH 040/105] Skip e2e, r2

---
 .github/workflows/build-test-reusable.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 92563b9bd9..6b59668ee6 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -200,7 +200,7 @@ jobs:
           env_manager: ${{ inputs.env-manager }}
 
       - name: Run E2E test
-        if: inputs.env_manager == 'base' && nputs.python_version != '3.12'
+        if: inputs.env_manager == 'base' && inputs.python_version != '3.12'
         run: |
           # Set WORKSPACE for inductor_xpu_test.sh to make sure it creates "inductor_log" outside of pytorch cloned directory
           export WORKSPACE=$GITHUB_WORKSPACE

From 47e0d31488e2ffe6e313a6c68ef53b36f3595526 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Wed, 9 Oct 2024 20:12:41 +0200
Subject: [PATCH 041/105] Fix sycl, instrumentation skiplist

---
 scripts/run_no_basekit.sh                  | 10 +++-------
 scripts/skiplist/conda/instrumentation.txt |  1 +
 2 files changed, 4 insertions(+), 7 deletions(-)
 create mode 100644 scripts/skiplist/conda/instrumentation.txt

diff --git a/scripts/run_no_basekit.sh b/scripts/run_no_basekit.sh
index 06856204cd..ed7692734e 100644
--- a/scripts/run_no_basekit.sh
+++ b/scripts/run_no_basekit.sh
@@ -13,13 +13,9 @@ install_conda() {
     find /opt/intel/oneapi/compiler/2024.1/lib/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
 
     wget https://files.pythonhosted.org/packages/cc/1e/d74e608f0c040e4f72dbfcd3b183f39570f054d08de39cc431f153220d90/intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
-    wheel unpack intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
-    mkdir -p ./intel_sycl_rt-2024.1.2/intel_sycl_rt-2024.1.2.data/data/include
-    cp -r /opt/intel/oneapi/compiler/2024.1/include/sycl ./intel_sycl_rt-2024.*/intel_sycl_rt-2024.*.data/data/include/
-    wheel pack intel_sycl_rt-2024.1.2 --build headers_patch
-    mv intel_sycl_rt-2024.1.2-headers_patch-py2.py3-none-manylinux1_x86_64.whl intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
-    pip install --force-reinstall ./intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
-    pip install dpcpp_cpp_rt==2024.1.2
+    pip install ./intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl dpcpp_cpp_rt==2024.1.2
+
+    ln -snf $HOME/miniconda3/envs/dpcpp/include/sycl $HOME/miniconda3/envs/triton/x86_64-conda-linux-gnu/sysroot/usr/include/
 }
 
 script_dir=$(dirname "$0")
diff --git a/scripts/skiplist/conda/instrumentation.txt b/scripts/skiplist/conda/instrumentation.txt
new file mode 100644
index 0000000000..aa206707a1
--- /dev/null
+++ b/scripts/skiplist/conda/instrumentation.txt
@@ -0,0 +1 @@
+test/unit/instrumentation/test_gpuhello

From 79e569d93b1fcaa5dda309e14553f872ce76ef76 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Thu, 10 Oct 2024 12:31:56 +0200
Subject: [PATCH 042/105] Remove some FIXME

---
 .github/workflows/no-basekit-build-test.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/no-basekit-build-test.yml b/.github/workflows/no-basekit-build-test.yml
index 314ebc88d5..63d647bdea 100644
--- a/.github/workflows/no-basekit-build-test.yml
+++ b/.github/workflows/no-basekit-build-test.yml
@@ -88,8 +88,7 @@ jobs:
       pytorch_mode: ${{ inputs.pytorch_mode || 'source' }}
       python_version: ${{ matrix.python }}
       upload_test_reports: ${{ inputs.upload_test_reports || false }}
-      ignore_errors: true
-      #FIXME${{ inputs.ignore_errors || false }}
+      ignore_errors: ${{ inputs.ignore_errors || false }}
       skip_list: no-basekit
       run_name: ${{ inputs.run_name }}
       enable_unskip: ${{ inputs.enable_unskip || false }}

From 077489ace7ab16918b787be67b00442259501471 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Thu, 10 Oct 2024 12:49:14 +0200
Subject: [PATCH 043/105] Remove some FIXME, r1

---
 .github/workflows/conda-build-test.yml      | 2 --
 .github/workflows/no-basekit-build-test.yml | 2 --
 2 files changed, 4 deletions(-)

diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml
index 0c0223645e..95e2e9b9be 100644
--- a/.github/workflows/conda-build-test.yml
+++ b/.github/workflows/conda-build-test.yml
@@ -43,8 +43,6 @@ on:
   pull_request:
     branches:
       - main
-  #FIXME      
-  push:
 
 permissions: read-all
 
diff --git a/.github/workflows/no-basekit-build-test.yml b/.github/workflows/no-basekit-build-test.yml
index 63d647bdea..5dce391ce4 100644
--- a/.github/workflows/no-basekit-build-test.yml
+++ b/.github/workflows/no-basekit-build-test.yml
@@ -43,8 +43,6 @@ on:
   pull_request:
     branches:
       - main
-  #FIXME      
-  push:
 
 permissions: read-all
 

From 378e3fa6692a963daae9277608ec4562476dbd55 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Thu, 10 Oct 2024 19:17:19 +0200
Subject: [PATCH 044/105] Update skiplist after rebase

---
 scripts/skiplist/conda/language.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/skiplist/conda/language.txt b/scripts/skiplist/conda/language.txt
index 4c618d732f..72711a2759 100644
--- a/scripts/skiplist/conda/language.txt
+++ b/scripts/skiplist/conda/language.txt
@@ -338,3 +338,4 @@ test/unit/language/test_core.py::test_const[ternary-False-False]
 test/unit/language/test_core.py::test_const[if-False-True]
 test/unit/language/test_core.py::test_const[if-False-False]
 test/unit/language/test_core.py::test_unroll_attr
+test/unit/language/test_decorator.py::test_triton_heuristic

From 7c55448f98e7314831e135aa7f8b23cc96273108 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Thu, 10 Oct 2024 19:31:29 +0200
Subject: [PATCH 045/105] Add nobasekit debug

---
 scripts/run_no_basekit.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scripts/run_no_basekit.sh b/scripts/run_no_basekit.sh
index ed7692734e..867672032d 100644
--- a/scripts/run_no_basekit.sh
+++ b/scripts/run_no_basekit.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -vx
+set -vxe
 
 install_conda() {
     conda create -n triton --override-channels -c conda-forge python=$python_version.*
@@ -15,7 +15,11 @@ install_conda() {
     wget https://files.pythonhosted.org/packages/cc/1e/d74e608f0c040e4f72dbfcd3b183f39570f054d08de39cc431f153220d90/intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
     pip install ./intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl dpcpp_cpp_rt==2024.1.2
 
+
+    #FIXME
     ln -snf $HOME/miniconda3/envs/dpcpp/include/sycl $HOME/miniconda3/envs/triton/x86_64-conda-linux-gnu/sysroot/usr/include/
+    find $HOME/miniconda3/envs/triton
+    exit 1
 }
 
 script_dir=$(dirname "$0")

From 67b71d8c5094f1910a99f5e7dc74fc448047e027 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Thu, 10 Oct 2024 19:54:19 +0200
Subject: [PATCH 046/105] Place sycl includes differently, r1

---
 scripts/run_no_basekit.sh | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/scripts/run_no_basekit.sh b/scripts/run_no_basekit.sh
index 867672032d..b736bfbed9 100644
--- a/scripts/run_no_basekit.sh
+++ b/scripts/run_no_basekit.sh
@@ -15,11 +15,7 @@ install_conda() {
     wget https://files.pythonhosted.org/packages/cc/1e/d74e608f0c040e4f72dbfcd3b183f39570f054d08de39cc431f153220d90/intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
     pip install ./intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl dpcpp_cpp_rt==2024.1.2
 
-
-    #FIXME
-    ln -snf $HOME/miniconda3/envs/dpcpp/include/sycl $HOME/miniconda3/envs/triton/x86_64-conda-linux-gnu/sysroot/usr/include/
-    find $HOME/miniconda3/envs/triton
-    exit 1
+    ln -snf /opt/intel/oneapi/compiler/2024.1/include/sycl $HOME/miniconda3/envs/include/
 }
 
 script_dir=$(dirname "$0")

From 0f1c18eb2c1d69aea80c4eece0c82cab9bf0b91c Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Thu, 10 Oct 2024 20:04:30 +0200
Subject: [PATCH 047/105] Place sycl includes differently, r2

---
 scripts/run_no_basekit.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/run_no_basekit.sh b/scripts/run_no_basekit.sh
index b736bfbed9..ea3c3b44d2 100644
--- a/scripts/run_no_basekit.sh
+++ b/scripts/run_no_basekit.sh
@@ -15,7 +15,7 @@ install_conda() {
     wget https://files.pythonhosted.org/packages/cc/1e/d74e608f0c040e4f72dbfcd3b183f39570f054d08de39cc431f153220d90/intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
     pip install ./intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl dpcpp_cpp_rt==2024.1.2
 
-    ln -snf /opt/intel/oneapi/compiler/2024.1/include/sycl $HOME/miniconda3/envs/include/
+    ln -snf /opt/intel/oneapi/compiler/2024.1/include/sycl $HOME/miniconda3/envs/triton/include/
 }
 
 script_dir=$(dirname "$0")

From 177a344bf16118c8dca8b87c593eea1422552acc Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Fri, 11 Oct 2024 16:21:23 +0200
Subject: [PATCH 048/105] Fix the skiplist

---
 .github/workflows/conda-build-test.yml      | 3 +++
 .github/workflows/no-basekit-build-test.yml | 2 ++
 scripts/skiplist/conda/runtime.txt          | 6 ++++++
 3 files changed, 11 insertions(+)

diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml
index 95e2e9b9be..b381249e4b 100644
--- a/.github/workflows/conda-build-test.yml
+++ b/.github/workflows/conda-build-test.yml
@@ -44,6 +44,9 @@ on:
     branches:
       - main
 
+  #FIXME
+  push:
+
 permissions: read-all
 
 jobs:
diff --git a/.github/workflows/no-basekit-build-test.yml b/.github/workflows/no-basekit-build-test.yml
index 5dce391ce4..3ed682b220 100644
--- a/.github/workflows/no-basekit-build-test.yml
+++ b/.github/workflows/no-basekit-build-test.yml
@@ -43,6 +43,8 @@ on:
   pull_request:
     branches:
       - main
+  #FIXME
+  push:
 
 permissions: read-all
 
diff --git a/scripts/skiplist/conda/runtime.txt b/scripts/skiplist/conda/runtime.txt
index f7a443f7d6..0f1d1cb4fb 100644
--- a/scripts/skiplist/conda/runtime.txt
+++ b/scripts/skiplist/conda/runtime.txt
@@ -1 +1,7 @@
 test/unit/runtime/test_bindings.py::test_module_walk
+test/unit/runtime/test_autotuner.py::test_kwargs[False]
+test/unit/runtime/test_autotuner.py::test_restore
+test/unit/runtime/test_autotuner.py::test_hooks
+test/unit/runtime/test_autotuner.py::test_prune_configs[False]
+test/unit/runtime/test_autotuner.py::test_prune_configs[True]
+test/unit/runtime/test_cache.py::test_jit_debug

From 068d215b03b37a25244fdc1d55545ac051ee2abc Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Fri, 11 Oct 2024 16:39:34 +0200
Subject: [PATCH 049/105] Fix sycl location

---
 scripts/run_no_basekit.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/run_no_basekit.sh b/scripts/run_no_basekit.sh
index ea3c3b44d2..85c3458133 100644
--- a/scripts/run_no_basekit.sh
+++ b/scripts/run_no_basekit.sh
@@ -15,7 +15,7 @@ install_conda() {
     wget https://files.pythonhosted.org/packages/cc/1e/d74e608f0c040e4f72dbfcd3b183f39570f054d08de39cc431f153220d90/intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
     pip install ./intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl dpcpp_cpp_rt==2024.1.2
 
-    ln -snf /opt/intel/oneapi/compiler/2024.1/include/sycl $HOME/miniconda3/envs/triton/include/
+    ln -snf /opt/intel/oneapi/compiler/2024.1/include/sycl $HOME/miniforge3/envs/triton/x86_64-conda-linux-gnu/sysroot/usr/include/sycl
 }
 
 script_dir=$(dirname "$0")

From 3895ae4c9b047538f0728ac495b1ece58b8c4164 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Fri, 11 Oct 2024 16:49:21 +0200
Subject: [PATCH 050/105] Fix the skiplist

---
 scripts/skiplist/conda/regression.txt | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 scripts/skiplist/conda/regression.txt

diff --git a/scripts/skiplist/conda/regression.txt b/scripts/skiplist/conda/regression.txt
new file mode 100644
index 0000000000..b7f0dada3d
--- /dev/null
+++ b/scripts/skiplist/conda/regression.txt
@@ -0,0 +1 @@
+python/test/regression/test_functional_regressions.py::test_reverse_range

From 15a01cf5554d922991d3a20b86237969afbd720a Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Fri, 11 Oct 2024 17:07:33 +0200
Subject: [PATCH 051/105] Add no_basekit debug

---
 scripts/run_no_basekit.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/run_no_basekit.sh b/scripts/run_no_basekit.sh
index 85c3458133..7d6d5f9f0a 100644
--- a/scripts/run_no_basekit.sh
+++ b/scripts/run_no_basekit.sh
@@ -15,7 +15,9 @@ install_conda() {
     wget https://files.pythonhosted.org/packages/cc/1e/d74e608f0c040e4f72dbfcd3b183f39570f054d08de39cc431f153220d90/intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
     pip install ./intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl dpcpp_cpp_rt==2024.1.2
 
-    ln -snf /opt/intel/oneapi/compiler/2024.1/include/sycl $HOME/miniforge3/envs/triton/x86_64-conda-linux-gnu/sysroot/usr/include/sycl
+    ln -snf /opt/intel/oneapi/compiler/2024.1/include/sycl $HOME/miniforge3/envs/triton/include/sycl
+    ls -l $HOME/miniforge3/envs/triton/include/sycl/
+    exit 1
 }
 
 script_dir=$(dirname "$0")

From ada771272c87ed809ab001746e98ce98a77e73ea Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Fri, 11 Oct 2024 17:34:09 +0200
Subject: [PATCH 052/105] Fix a skiplist, r

---
 scripts/skiplist/conda/regression.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/skiplist/conda/regression.txt b/scripts/skiplist/conda/regression.txt
index b7f0dada3d..8641b2f4ae 100644
--- a/scripts/skiplist/conda/regression.txt
+++ b/scripts/skiplist/conda/regression.txt
@@ -1 +1 @@
-python/test/regression/test_functional_regressions.py::test_reverse_range
+test/regression/test_functional_regressions.py::test_reverse_range

From 6da4c5e4cf309528e6c337c35b181f6b6c097d17 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Fri, 11 Oct 2024 17:34:28 +0200
Subject: [PATCH 053/105] Fix a skiplist, r2

---
 .github/workflows/no-basekit-build-test.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/no-basekit-build-test.yml b/.github/workflows/no-basekit-build-test.yml
index 3ed682b220..5dce391ce4 100644
--- a/.github/workflows/no-basekit-build-test.yml
+++ b/.github/workflows/no-basekit-build-test.yml
@@ -43,8 +43,6 @@ on:
   pull_request:
     branches:
       - main
-  #FIXME
-  push:
 
 permissions: read-all
 

From 5a9a51d0043c5e1cbc7ed51c9c5e267b329c2d6b Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Fri, 11 Oct 2024 18:04:13 +0200
Subject: [PATCH 054/105] Remove no_basekit changes

---
 .github/workflows/no-basekit-build-test.yml | 202 ++++++++++++--------
 scripts/run_no_basekit.sh                   |  34 ----
 2 files changed, 123 insertions(+), 113 deletions(-)
 delete mode 100644 scripts/run_no_basekit.sh

diff --git a/.github/workflows/no-basekit-build-test.yml b/.github/workflows/no-basekit-build-test.yml
index 5dce391ce4..dfa0313cbb 100644
--- a/.github/workflows/no-basekit-build-test.yml
+++ b/.github/workflows/no-basekit-build-test.yml
@@ -1,93 +1,137 @@
-name: No-Basekit Test
-run-name: ${{ inputs.run_name }}
+name: No-basekit build and test
 
 on:
   workflow_dispatch:
-    inputs:
-      runner_label:
-        description: Runner label, keep empty for default
-        type: string
-        default: ""
-      pytorch_ref:
-        description: PyTorch ref, keep empty for default
-        type: string
-        default: ""
-      pytorch_mode:
-        description: PyTorch mode, source or wheels
-        type: choice
-        options:
-          - source
-          - wheels
-        default: source
-      upload_test_reports:
-        description: Upload test reports
-        type: boolean
-        default: false
-      ignore_errors:
-        description: Ignore test errors
-        type: boolean
-        default: false
-      skip_list:
-        description: Skip list
-        type: string
-        default: ""
-      run_name:
-        description: Custom run name
-        type: string
-        default: "Build and test"
-      enable_unskip:
-        description: Ignore pytest.skip
-        type: boolean
-        default: false
-
-  pull_request:
-    branches:
-      - main
+  schedule:
+    - cron: "5 2 * * *"
 
 permissions: read-all
 
-jobs:
-  prepare:
-    name: Prepare
-    runs-on: Linux
+env:
+  TRITON_DISABLE_LINE_INFO: 1
 
-    outputs:
-      matrix: ${{ steps.matrix.outputs.matrix }}
+jobs:
 
+  integration-tests:
+    name: Integration tests
+    runs-on:
+      - max1100
+      - rolling
+      - runner-0.0.19
+    strategy:
+      matrix:
+        python: ${{ github.ref_name == 'main' && fromJson('["3.9", "3.10", "3.11"]') || fromJson('["3.9"]') }}
+    defaults:
+      run:
+        shell: bash -noprofile --norc -eo pipefail -c "source {0}"
     steps:
-      - name: Inputs
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Calculate PATH
         run: |
-          cat <<EOF
-          ${{ toJSON(inputs) }}
-          EOF
+          echo $HOME/miniforge3/bin >>$GITHUB_PATH
+
+      - name: Load conda cache
+        id: conda-cache
+        uses: ./.github/actions/load
+        env:
+          CACHE_NUMBER: 6
+        with:
+          path: $HOME/miniforge3/envs/triton
+          key: no-basekit-py${{ matrix.python }}-${{ hashFiles('scripts/triton.yml', 'python/pyproject.toml', 'python/setup.py', '.github/pins/ipex.txt', '.github/pins/pytorch.txt') }}-${{ env.CACHE_NUMBER }}
 
-      - name: Matrix
-        id: matrix
+      - name: Update conda env
+        if: ${{ steps.conda-cache.outputs.status == 'miss' }}
         run: |
-          if [[ -n "${{ inputs.runner_label }}" ]]; then
-            matrix='{"python": ["3.9"]}'
-          else
-            matrix='{"python": ["3.9"], "driver": ["rolling", "lts"]}'
-          fi
-          echo "matrix=$matrix" | tee -a $GITHUB_OUTPUT
+          conda create -n triton --override-channels -c conda-forge python=${{ matrix.python }}.*
+          conda env update -f scripts/triton.yml
 
-  integration-tests:
-    name: Integration tests matrix
-    needs: prepare
+          ln -snf /usr/include/level_zero $HOME/miniforge3/envs/triton/x86_64-conda-linux-gnu/sysroot/usr/include/level_zero
+          find /usr -name libze_\* -exec ln -sf {} $HOME/miniforge3/envs/triton/lib/ \;
 
-    strategy:
-      matrix: ${{ fromJson(needs.prepare.outputs.matrix) }}
-
-    uses: ./.github/workflows/build-test-reusable.yml
-    with:
-      driver_version: ${{ matrix.driver }}
-      runner_label: ${{ inputs.runner_label }}
-      pytorch_ref: ${{ inputs.pytorch_ref }}
-      pytorch_mode: ${{ inputs.pytorch_mode || 'source' }}
-      python_version: ${{ matrix.python }}
-      upload_test_reports: ${{ inputs.upload_test_reports || false }}
-      ignore_errors: ${{ inputs.ignore_errors || false }}
-      skip_list: no-basekit
-      run_name: ${{ inputs.run_name }}
-      enable_unskip: ${{ inputs.enable_unskip || false }}
-      env_manager: no_basekit
+          find /opt/intel/oneapi/mkl/2025.0/lib/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
+          find /opt/intel/oneapi/compiler/2024.1/lib/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
+
+      - name: Calculate ENV
+        run: |
+          python -m venv ./.venv; source ./.venv/bin/activate
+          echo LD_LIBRARY_PATH=$HOME/miniforge3/envs/triton/lib:$LD_LIBRARY_PATH:$VIRTUAL_ENV/lib >>$GITHUB_ENV
+          echo CPATH=$CPATH:$VIRTUAL_ENV/include:$VIRTUAL_ENV/include/sycl >>$GITHUB_ENV
+
+      - name: Install deps
+        run: |
+          python -m venv ./.venv; source ./.venv/bin/activate
+
+          wget https://files.pythonhosted.org/packages/cc/1e/d74e608f0c040e4f72dbfcd3b183f39570f054d08de39cc431f153220d90/intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
+          wheel unpack intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
+          mkdir -p ./intel_sycl_rt-2024.1.2/intel_sycl_rt-2024.1.2.data/data/include
+          cp -r /opt/intel/oneapi/compiler/2024.1/include/sycl ./intel_sycl_rt-2024.*/intel_sycl_rt-2024.*.data/data/include/
+          wheel pack intel_sycl_rt-2024.1.2 --build headers_patch
+          mv intel_sycl_rt-2024.1.2-headers_patch-py2.py3-none-manylinux1_x86_64.whl intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
+          pip install --force-reinstall ./intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
+          pip install dpcpp_cpp_rt==2024.1.2
+
+
+      - name: Add conda info to log
+        run: |
+          conda info
+          conda list -n triton
+
+      - name: Install latest nightly wheels
+        uses: ./.github/actions/install-wheels
+        with:
+          gh_token: ${{ secrets.GITHUB_TOKEN }}
+          install_cmd: conda run --no-capture-output -n triton pip install
+          python_version: ${{ matrix.python }}
+          wheels_pattern: 'torch-*'
+
+      - name: Build Triton
+        run: |
+          set -x
+          python -m venv ./.venv; source ./.venv/bin/activate
+          export DEBUG=1
+          cd python
+          conda run --no-capture-output -n triton pip install pybind11
+          conda run --no-capture-output -n triton pip install --no-build-isolation -e '.[build,tests,tutorials]'
+
+      - name: Run tests
+        run: |
+          set -x
+          python -m venv ./.venv; source ./.venv/bin/activate
+
+          export TRITON_TEST_REPORTS=true
+          export TRITON_TEST_REPORTS_DIR=$HOME/reports
+          # FIXME https://github.com/intel/intel-xpu-backend-for-triton/issues/866
+          export TRITON_TEST_SKIPLIST_DIR=scripts/skiplist/no-basekit
+          conda run --no-capture-output -n triton bash -v -x scripts/test-triton.sh
+
+      - name: Run E2E test
+        run: |
+          # FIXME https://github.com/intel/intel-xpu-backend-for-triton/issues/1031
+          cd ../pytorch || {
+            PYTORCH_COMMIT_ID=$(<.github/pins/pytorch.txt)
+            cd ..
+            git clone --single-branch -b dev/triton-test-3.0 --recurse-submodules https://github.com/Stonepia/pytorch.git
+            cd pytorch
+            git branch pin-branch $PYTORCH_COMMIT_ID
+            git switch pin-branch
+          }
+
+          TRANSFORMERS_VERSION="$(<.ci/docker/ci_commit_pins/huggingface.txt)"
+          conda run -n triton pip install pyyaml pandas scipy numpy psutil pyre_extensions torchrec transformers==$TRANSFORMERS_VERSION
+
+          # Set WORKSPACE for inductor_xpu_test.sh to make sure it creates "inductor_log" outside of pytorch cloned directory
+          export WORKSPACE=$GITHUB_WORKSPACE
+          # TODO: Find the fastest Hugging Face model
+          conda run --no-capture-output -n triton $GITHUB_WORKSPACE/scripts/inductor_xpu_test.sh huggingface float32 inference accuracy xpu 0 static 1 0 AlbertForMaskedLM
+          # The script above always returns 0, so we need an additional check to see if the accuracy test passed
+          cat $WORKSPACE/inductor_log/*/*/*.csv
+          grep AlbertForMaskedLM $WORKSPACE/inductor_log/*/*/*.csv | grep -q ,pass,
+
+      - name: Save conda cache
+        if: ${{ steps.conda-cache.outputs.status == 'miss' }}
+        uses: ./.github/actions/save
+        with:
+          path: ${{ steps.conda-cache.outputs.path }}
+          dest: ${{ steps.conda-cache.outputs.dest }}
diff --git a/scripts/run_no_basekit.sh b/scripts/run_no_basekit.sh
deleted file mode 100644
index 7d6d5f9f0a..0000000000
--- a/scripts/run_no_basekit.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-
-set -vxe
-
-install_conda() {
-    conda create -n triton --override-channels -c conda-forge python=$python_version.*
-    conda env update -f scripts/triton.yml
-
-    ln -snf /usr/include/level_zero $HOME/miniforge3/envs/triton/x86_64-conda-linux-gnu/sysroot/usr/include/level_zero
-    find /usr -name libze_\* -exec ln -sf {} $HOME/miniforge3/envs/triton/lib/ \;
-
-    find /opt/intel/oneapi/mkl/2025.0/lib/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
-    find /opt/intel/oneapi/compiler/2024.1/lib/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
-
-    wget https://files.pythonhosted.org/packages/cc/1e/d74e608f0c040e4f72dbfcd3b183f39570f054d08de39cc431f153220d90/intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl
-    pip install ./intel_sycl_rt-2024.1.2-py2.py3-none-manylinux1_x86_64.whl dpcpp_cpp_rt==2024.1.2
-
-    ln -snf /opt/intel/oneapi/compiler/2024.1/include/sycl $HOME/miniforge3/envs/triton/include/sycl
-    ls -l $HOME/miniforge3/envs/triton/include/sycl/
-    exit 1
-}
-
-script_dir=$(dirname "$0")
-source "$script_dir/run_util.sh"
-
-export PATH="$HOME/miniforge3/bin:$PATH"
-test -d "$HOME/miniforge3/envs/triton" || install_conda
-print_conda_info
-
-python -m venv ./.venv; source ./.venv/bin/activate
-export LD_LIBRARY_PATH=$HOME/miniforge3/envs/triton/lib:$LD_LIBRARY_PATH:$VIRTUAL_ENV/lib
-export CPATH=$CPATH:$VIRTUAL_ENV/include:$VIRTUAL_ENV/include/sycl
-
-conda run --no-capture-output -n triton bash "$script_name"

From 769cf96937fe25d61525b534e69ec8b74486c701 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Fri, 11 Oct 2024 18:05:23 +0200
Subject: [PATCH 055/105] Remove FIXME

---
 .github/workflows/conda-build-test.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml
index b381249e4b..95e2e9b9be 100644
--- a/.github/workflows/conda-build-test.yml
+++ b/.github/workflows/conda-build-test.yml
@@ -44,9 +44,6 @@ on:
     branches:
       - main
 
-  #FIXME
-  push:
-
 permissions: read-all
 
 jobs:

From 5a258a243ead7c45b1b294ca9eee75310f18d2d8 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Fri, 11 Oct 2024 19:12:27 +0200
Subject: [PATCH 056/105] Change underscore to minus in script names, add +x

---
 .github/workflows/build-test-reusable.yml | 2 +-
 scripts/{run_base.sh => run-base.sh}      | 0
 scripts/{run_conda.sh => run-conda.sh}    | 2 +-
 scripts/{run_util.sh => run-util.sh}      | 2 --
 4 files changed, 2 insertions(+), 4 deletions(-)
 rename scripts/{run_base.sh => run-base.sh} (100%)
 mode change 100644 => 100755
 rename scripts/{run_conda.sh => run-conda.sh} (95%)
 mode change 100644 => 100755
 rename scripts/{run_util.sh => run-util.sh} (95%)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 6b59668ee6..4a3e62fd4b 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -73,7 +73,7 @@ jobs:
     runs-on: ${{ fromJson(inputs.runner_label && format('["{0}"]', inputs.runner_label) || format('["{0}", "{1}", "{2}"]', inputs.device, inputs.driver_version, inputs.runner_version)) }}
     defaults:
       run:
-        shell: bash -noprofile --norc -eo pipefail scripts/run_${{ inputs.env_manager }}.sh --python-version ${{ inputs.python_version }} {0}
+        shell: bash -noprofile --norc -eo pipefail scripts/run-${{ inputs.env_manager }}.sh --python-version ${{ inputs.python_version }} {0}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
diff --git a/scripts/run_base.sh b/scripts/run-base.sh
old mode 100644
new mode 100755
similarity index 100%
rename from scripts/run_base.sh
rename to scripts/run-base.sh
diff --git a/scripts/run_conda.sh b/scripts/run-conda.sh
old mode 100644
new mode 100755
similarity index 95%
rename from scripts/run_conda.sh
rename to scripts/run-conda.sh
index 40f2100d98..06ae578f38
--- a/scripts/run_conda.sh
+++ b/scripts/run-conda.sh
@@ -15,7 +15,7 @@ source /opt/intel/oneapi/setvars.sh >/dev/null
 set -vx
 
 script_dir=$(dirname "$0")
-source "$script_dir/run_util.sh"
+source "$script_dir/run-util.sh"
 
 export PATH="$HOME/miniforge3/bin:$PATH"
 test -d "$HOME/miniforge3/envs/triton" || install_conda
diff --git a/scripts/run_util.sh b/scripts/run-util.sh
similarity index 95%
rename from scripts/run_util.sh
rename to scripts/run-util.sh
index 719477dcbf..28b23f337d 100644
--- a/scripts/run_util.sh
+++ b/scripts/run-util.sh
@@ -1,5 +1,3 @@
-#!/bin/bash
-
 print_conda_info() {
     test ! -f first_run || return 0
     touch first_run

From 6a7898dcd9f6a588c5844c9cf6c68742f8c9f489 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Fri, 11 Oct 2024 19:14:28 +0200
Subject: [PATCH 057/105] Change underscore to minus in script names, add +x,
 r1

---
 scripts/run-base.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/run-base.sh b/scripts/run-base.sh
index 7379f27053..9dc1a8ba7e 100755
--- a/scripts/run-base.sh
+++ b/scripts/run-base.sh
@@ -3,5 +3,5 @@
 source /opt/intel/oneapi/setvars.sh >/dev/null
 
 script_dir=$(dirname "$0")
-source "$script_dir/run_util.sh"
+source "$script_dir/run-util.sh"
 source "$script_name"

From ecf7c697fe6f65cefb71107c1ce39a1de8479bc1 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 14 Oct 2024 16:50:21 +0200
Subject: [PATCH 058/105] Split env installation into a separate step

---
 .github/workflows/build-test-reusable.yml | 4 ++++
 scripts/run-base.sh                       | 4 ++++
 scripts/run-conda.sh                      | 6 +-----
 scripts/run-util.sh                       | 9 ++++++---
 4 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 4a3e62fd4b..cb55fb6fe4 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -88,6 +88,10 @@ jobs:
           path: $HOME/miniforge3/envs/triton
           key: conda-${{ inputs.env_manager }}-py${{ matrix.python }}-${{ hashFiles('scripts/triton.yml', 'python/pyproject.toml', 'python/setup.py') }}-${{ env.CACHE_NUMBER }}
 
+      - name: Install Manager Environment
+        run: |
+          install-env
+
       - name: Print inputs
         run: |
           cat <<EOF
diff --git a/scripts/run-base.sh b/scripts/run-base.sh
index 9dc1a8ba7e..3761ca38e1 100755
--- a/scripts/run-base.sh
+++ b/scripts/run-base.sh
@@ -1,5 +1,9 @@
 #!/bin/bash
 
+install_env() {
+  :
+}
+
 source /opt/intel/oneapi/setvars.sh >/dev/null
 
 script_dir=$(dirname "$0")
diff --git a/scripts/run-conda.sh b/scripts/run-conda.sh
index 06ae578f38..200efdfef3 100755
--- a/scripts/run-conda.sh
+++ b/scripts/run-conda.sh
@@ -1,13 +1,11 @@
 #!/bin/bash
 
-install_conda() {
+install_env() {
     conda create -n triton --override-channels -c conda-forge python=$python_version.*
     conda env update -f scripts/triton.yml
     find /opt/intel/oneapi/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
     ln -snf /usr/include/level_zero $HOME/miniforge3/envs/triton/bin/../x86_64-conda-linux-gnu/sysroot/usr/include/level_zero
     find /usr -name libze_\* -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
-    conda info
-    conda list -n triton
 }
 
 source /opt/intel/oneapi/setvars.sh >/dev/null
@@ -18,7 +16,5 @@ script_dir=$(dirname "$0")
 source "$script_dir/run-util.sh"
 
 export PATH="$HOME/miniforge3/bin:$PATH"
-test -d "$HOME/miniforge3/envs/triton" || install_conda
-print_conda_info
 
 conda run --no-capture-output -n triton bash "$script_name"
diff --git a/scripts/run-util.sh b/scripts/run-util.sh
index 28b23f337d..07bb245fb4 100644
--- a/scripts/run-util.sh
+++ b/scripts/run-util.sh
@@ -1,6 +1,4 @@
-print_conda_info() {
-    test ! -f first_run || return 0
-    touch first_run
+print_env_info() {
     conda info
     conda list -n triton
 }
@@ -11,6 +9,11 @@ while [ -v 1 ]; do
       python_version=$2
       shift 2
       ;;
+    install-env)
+      install_env
+      print_env_info
+      exit 0
+      ;;
     *)
       script_name=$1
       shift

From 38f9b2530b64a004dc41ef10b1ee7c9116a1ae62 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 14 Oct 2024 17:08:12 +0200
Subject: [PATCH 059/105] Split env installation into a separate step, r1

---
 scripts/run-base.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/run-base.sh b/scripts/run-base.sh
index 3761ca38e1..f15155ba9f 100755
--- a/scripts/run-base.sh
+++ b/scripts/run-base.sh
@@ -5,7 +5,7 @@ install_env() {
 }
 
 source /opt/intel/oneapi/setvars.sh >/dev/null
-
+set -vx
 script_dir=$(dirname "$0")
 source "$script_dir/run-util.sh"
 source "$script_name"

From 6e7343bd5178be0d480434fb425b25d15a1b65ca Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 14 Oct 2024 17:16:41 +0200
Subject: [PATCH 060/105] Split env installation into a separate step, r2

---
 scripts/run-util.sh | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/scripts/run-util.sh b/scripts/run-util.sh
index 07bb245fb4..bd5a1a802a 100644
--- a/scripts/run-util.sh
+++ b/scripts/run-util.sh
@@ -2,21 +2,23 @@ print_env_info() {
     conda info
     conda list -n triton
 }
-
+set -vx
 while [ -v 1 ]; do
   case "$1" in
     --python-version)
       python_version=$2
       shift 2
       ;;
-    install-env)
-      install_env
-      print_env_info
-      exit 0
-      ;;
     *)
       script_name=$1
       shift
       ;;
   esac
 done
+
+if [ $(head -1 "$script_name") = install-env ]
+then
+  install_env
+  print_env_info
+  exit 0
+fi

From 258afb3afde705eaeebe2936212ed54d22553c4d Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 14 Oct 2024 17:36:21 +0200
Subject: [PATCH 061/105] Split env installation into a separate step, r3

---
 .github/workflows/build-test-reusable.yml |  3 ++-
 scripts/install-base.sh                   |  3 +++
 scripts/install-conda.sh                  | 21 +++++++++++++++++++++
 scripts/run-base.sh                       |  6 +-----
 scripts/run-conda.sh                      | 10 ----------
 scripts/run-util.sh                       | 11 -----------
 6 files changed, 27 insertions(+), 27 deletions(-)
 create mode 100644 scripts/install-base.sh
 create mode 100644 scripts/install-conda.sh

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index cb55fb6fe4..ad23654504 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -89,8 +89,9 @@ jobs:
           key: conda-${{ inputs.env_manager }}-py${{ matrix.python }}-${{ hashFiles('scripts/triton.yml', 'python/pyproject.toml', 'python/setup.py') }}-${{ env.CACHE_NUMBER }}
 
       - name: Install Manager Environment
+        shell: bash --noprofile --norc -eo pipefail {0}
         run: |
-          install-env
+          scripts/install-${{ inputs.env_manager }}.sh --python-version ${{ inputs.python_version }}
 
       - name: Print inputs
         run: |
diff --git a/scripts/install-base.sh b/scripts/install-base.sh
new file mode 100644
index 0000000000..64768edc7b
--- /dev/null
+++ b/scripts/install-base.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo Nothing to install
diff --git a/scripts/install-conda.sh b/scripts/install-conda.sh
new file mode 100644
index 0000000000..97c49f1610
--- /dev/null
+++ b/scripts/install-conda.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+
+intall_env() {
+    conda create -n triton --override-channels -c conda-forge python=$python_version.*
+    conda env update -f scripts/triton.yml
+    find /opt/intel/oneapi/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
+    ln -snf /usr/include/level_zero $HOME/miniforge3/envs/triton/bin/../x86_64-conda-linux-gnu/sysroot/usr/include/level_zero
+    find /usr -name libze_\* -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
+}
+
+print_env_info() {
+    conda info
+    conda list -n triton
+}
+
+script_dir=$(dirname "$0")
+source "$script_dir/run-util.sh"
+
+install_env
+print_env_info
diff --git a/scripts/run-base.sh b/scripts/run-base.sh
index f15155ba9f..9dc1a8ba7e 100755
--- a/scripts/run-base.sh
+++ b/scripts/run-base.sh
@@ -1,11 +1,7 @@
 #!/bin/bash
 
-install_env() {
-  :
-}
-
 source /opt/intel/oneapi/setvars.sh >/dev/null
-set -vx
+
 script_dir=$(dirname "$0")
 source "$script_dir/run-util.sh"
 source "$script_name"
diff --git a/scripts/run-conda.sh b/scripts/run-conda.sh
index 200efdfef3..271900dd51 100755
--- a/scripts/run-conda.sh
+++ b/scripts/run-conda.sh
@@ -1,17 +1,7 @@
 #!/bin/bash
 
-install_env() {
-    conda create -n triton --override-channels -c conda-forge python=$python_version.*
-    conda env update -f scripts/triton.yml
-    find /opt/intel/oneapi/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
-    ln -snf /usr/include/level_zero $HOME/miniforge3/envs/triton/bin/../x86_64-conda-linux-gnu/sysroot/usr/include/level_zero
-    find /usr -name libze_\* -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
-}
-
 source /opt/intel/oneapi/setvars.sh >/dev/null
 
-set -vx
-
 script_dir=$(dirname "$0")
 source "$script_dir/run-util.sh"
 
diff --git a/scripts/run-util.sh b/scripts/run-util.sh
index bd5a1a802a..fbf2b7f737 100644
--- a/scripts/run-util.sh
+++ b/scripts/run-util.sh
@@ -1,8 +1,3 @@
-print_env_info() {
-    conda info
-    conda list -n triton
-}
-set -vx
 while [ -v 1 ]; do
   case "$1" in
     --python-version)
@@ -16,9 +11,3 @@ while [ -v 1 ]; do
   esac
 done
 
-if [ $(head -1 "$script_name") = install-env ]
-then
-  install_env
-  print_env_info
-  exit 0
-fi

From 4c644406ab72262dad95302ffec39d53993e35a3 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 14 Oct 2024 17:40:03 +0200
Subject: [PATCH 062/105] Split env installation into a separate step, r4

---
 scripts/install-base.sh  | 0
 scripts/install-conda.sh | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 scripts/install-base.sh
 mode change 100644 => 100755 scripts/install-conda.sh

diff --git a/scripts/install-base.sh b/scripts/install-base.sh
old mode 100644
new mode 100755
diff --git a/scripts/install-conda.sh b/scripts/install-conda.sh
old mode 100644
new mode 100755

From 60e14a919ddfa3df302a443b6bd9125824584a7c Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 14 Oct 2024 17:46:31 +0200
Subject: [PATCH 063/105] Split env installation into a separate step, r5

---
 scripts/install-conda.sh | 1 +
 scripts/run-util.sh      | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-conda.sh b/scripts/install-conda.sh
index 97c49f1610..1c6dbb8791 100755
--- a/scripts/install-conda.sh
+++ b/scripts/install-conda.sh
@@ -2,6 +2,7 @@
 
 
 intall_env() {
+    export PATH="$HOME/miniforge3/bin:$PATH"
     conda create -n triton --override-channels -c conda-forge python=$python_version.*
     conda env update -f scripts/triton.yml
     find /opt/intel/oneapi/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
diff --git a/scripts/run-util.sh b/scripts/run-util.sh
index fbf2b7f737..e5aeb4cf8e 100644
--- a/scripts/run-util.sh
+++ b/scripts/run-util.sh
@@ -10,4 +10,3 @@ while [ -v 1 ]; do
       ;;
   esac
 done
-

From 19b2c66488a9017cb1527e798fe84e1ced8a3102 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 14 Oct 2024 17:52:15 +0200
Subject: [PATCH 064/105] Split env installation into a separate step, r6

---
 scripts/{run-util.sh => env-util.sh} | 0
 scripts/install-conda.sh             | 5 ++---
 scripts/run-base.sh                  | 2 +-
 scripts/run-conda.sh                 | 2 +-
 4 files changed, 4 insertions(+), 5 deletions(-)
 rename scripts/{run-util.sh => env-util.sh} (100%)

diff --git a/scripts/run-util.sh b/scripts/env-util.sh
similarity index 100%
rename from scripts/run-util.sh
rename to scripts/env-util.sh
diff --git a/scripts/install-conda.sh b/scripts/install-conda.sh
index 1c6dbb8791..953b20f5f6 100755
--- a/scripts/install-conda.sh
+++ b/scripts/install-conda.sh
@@ -1,7 +1,6 @@
 #!/bin/bash
 
-
-intall_env() {
+install_env() {
     export PATH="$HOME/miniforge3/bin:$PATH"
     conda create -n triton --override-channels -c conda-forge python=$python_version.*
     conda env update -f scripts/triton.yml
@@ -16,7 +15,7 @@ print_env_info() {
 }
 
 script_dir=$(dirname "$0")
-source "$script_dir/run-util.sh"
+source "$script_dir/env-util.sh"
 
 install_env
 print_env_info
diff --git a/scripts/run-base.sh b/scripts/run-base.sh
index 9dc1a8ba7e..e3296b8d96 100755
--- a/scripts/run-base.sh
+++ b/scripts/run-base.sh
@@ -3,5 +3,5 @@
 source /opt/intel/oneapi/setvars.sh >/dev/null
 
 script_dir=$(dirname "$0")
-source "$script_dir/run-util.sh"
+source "$script_dir/env-util.sh"
 source "$script_name"
diff --git a/scripts/run-conda.sh b/scripts/run-conda.sh
index 271900dd51..1e26fa1b60 100755
--- a/scripts/run-conda.sh
+++ b/scripts/run-conda.sh
@@ -3,7 +3,7 @@
 source /opt/intel/oneapi/setvars.sh >/dev/null
 
 script_dir=$(dirname "$0")
-source "$script_dir/run-util.sh"
+source "$script_dir/env-util.sh"
 
 export PATH="$HOME/miniforge3/bin:$PATH"
 

From d3986c21e861f15b5c3485d453a2c3a4dab8105d Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 14 Oct 2024 20:13:53 +0200
Subject: [PATCH 065/105] Update a skiplist after rebase

---
 scripts/skiplist/conda/language.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/skiplist/conda/language.txt b/scripts/skiplist/conda/language.txt
index 72711a2759..b540a2a877 100644
--- a/scripts/skiplist/conda/language.txt
+++ b/scripts/skiplist/conda/language.txt
@@ -339,3 +339,4 @@ test/unit/language/test_core.py::test_const[if-False-True]
 test/unit/language/test_core.py::test_const[if-False-False]
 test/unit/language/test_core.py::test_unroll_attr
 test/unit/language/test_decorator.py::test_triton_heuristic
+test/unit/language/test_core.py::test_constexpr_if_return

From e9066578874d05df0d0a3e40f4659af0a37b4e24 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 14 Oct 2024 22:19:27 +0200
Subject: [PATCH 066/105] Remove python version from runner

---
 .github/workflows/build-test-reusable.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index ad23654504..4e92ad81e7 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -73,7 +73,7 @@ jobs:
     runs-on: ${{ fromJson(inputs.runner_label && format('["{0}"]', inputs.runner_label) || format('["{0}", "{1}", "{2}"]', inputs.device, inputs.driver_version, inputs.runner_version)) }}
     defaults:
       run:
-        shell: bash -noprofile --norc -eo pipefail scripts/run-${{ inputs.env_manager }}.sh --python-version ${{ inputs.python_version }} {0}
+        shell: bash -noprofile --norc -eo pipefail scripts/run-${{ inputs.env_manager }}.sh {0}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4

From 4220bf3a39743601fa5edb909f61d4f24c354a80 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 14 Oct 2024 23:18:33 +0200
Subject: [PATCH 067/105] Fix transformers install

---
 .github/workflows/build-test-reusable.yml | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index d1e8a132bf..1b341e8caa 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -144,8 +144,8 @@ jobs:
         run: |
           export DEBUG=1
           cd python
-          conda run --no-capture-output -n triton pip install pybind11
-          conda run --no-capture-output -n triton pip install --no-build-isolation -e '.[build,tests,tutorials]'
+          pip install pybind11
+          pip install --no-build-isolation -e '.[build,tests,tutorials]'
 
       - name: Create test-triton command line
         run: |
@@ -195,13 +195,22 @@ jobs:
           echo "TRANSFORMERS_VERSION=$TRANSFORMERS_VERSION" | tee -a $GITHUB_ENV
 
       - name: Install transformers
+        if: inputs.env_manager == 'base'
         uses: ./.github/actions/install-dependency
         with:
           package: transformers
           repository: huggingface/transformers
           ref: ${{ env.TRANSFORMERS_VERSION }}
           try-tag-prefix: v
-          env_manager: ${{ inputs.env-manager }}
+
+      - name: Install transformers
+        if: inputs.env_manager != 'base'
+        run:
+          git clone --recursive https://github.com/huggingface/transformers
+          cd transformers
+          git checkout ${{ env.TRANSFORMERS_VERSION }}
+          python setup.py bdist_wheel
+          pip install dist/*.whl
 
       - name: Run E2E test
         run: |

From 0d6ed878af97ee82c693dc9afbdd4b0716b805bd Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 14 Oct 2024 23:43:32 +0200
Subject: [PATCH 068/105] Fix transformers install, r1

---
 .github/workflows/build-test-reusable.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 1b341e8caa..68b1da7a85 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -205,7 +205,7 @@ jobs:
 
       - name: Install transformers
         if: inputs.env_manager != 'base'
-        run:
+        run: |
           git clone --recursive https://github.com/huggingface/transformers
           cd transformers
           git checkout ${{ env.TRANSFORMERS_VERSION }}

From ff85043191e62bab611b01d69f0fe3811b3bdf3b Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Tue, 15 Oct 2024 00:40:12 +0200
Subject: [PATCH 069/105] Fix pytorch install

---
 .github/workflows/build-test-reusable.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 68b1da7a85..346301c7bf 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -128,6 +128,9 @@ jobs:
         if: inputs.env_manager != 'base'
         run: |
           pip install torch --index-url https://download.pytorch.org/whl/nightly/xpu
+          git clone --recursive https://github.com/pytorch/pytorch
+          cd pytorch
+          git checkout ${{ inputs.pytorch_ref }}
 
       - name: Install pass_rate dependencies
         run: |
@@ -204,20 +207,21 @@ jobs:
           try-tag-prefix: v
 
       - name: Install transformers
-        if: inputs.env_manager != 'base'
+        if: (inputs.env_manager != 'base') && (steps.conda-cache.outputs.status == 'miss')
         run: |
           git clone --recursive https://github.com/huggingface/transformers
           cd transformers
           git checkout ${{ env.TRANSFORMERS_VERSION }}
           python setup.py bdist_wheel
           pip install dist/*.whl
+          python -c "import transformers; print(transformers.__version__)"
+          pip install pyyaml pandas scipy numpy psutil pyre_extensions torchrec
 
       - name: Run E2E test
         run: |
           # Set WORKSPACE for inductor_xpu_test.sh to make sure it creates "inductor_log" outside of pytorch cloned directory
           export WORKSPACE=$GITHUB_WORKSPACE
           cd pytorch
-          pip install pyyaml pandas scipy numpy psutil pyre_extensions torchrec
           # TODO: Find the fastest Hugging Face model
           $GITHUB_WORKSPACE/scripts/inductor_xpu_test.sh huggingface float32 inference accuracy xpu 0 static 1 0 AlbertForMaskedLM
           # The script above always returns 0, so we need an additional check to see if the accuracy test passed

From 0f94582de59969b35b5ad1995e47f1a5d05ef643 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Tue, 15 Oct 2024 01:35:16 +0200
Subject: [PATCH 070/105] Fix pytorch install, r1

---
 .github/workflows/build-test-reusable.yml | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 346301c7bf..343897388b 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -78,6 +78,13 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Clone PyTorch
+        if: inputs.env_manager != 'base'
+        run: |
+          git clone --recursive https://github.com/pytorch/pytorch
+          cd pytorch
+          git checkout ${{ inputs.pytorch_ref }}
+
       - name: Load conda cache
         if: inputs.env_manager != 'base'
         id: conda-cache
@@ -86,7 +93,7 @@ jobs:
           CACHE_NUMBER: 6
         with:
           path: $HOME/miniforge3/envs/triton
-          key: conda-${{ inputs.env_manager }}-py${{ matrix.python }}-${{ hashFiles('scripts/triton.yml', 'python/pyproject.toml', 'python/setup.py') }}-${{ env.CACHE_NUMBER }}
+          key: conda-${{ inputs.env_manager }}-py${{ matrix.python }}-${{ hashFiles('scripts/triton.yml', 'python/pyproject.toml', 'python/setup.py', 'pytorch/.ci/docker/ci_commit_pins/huggingface.txt') }}-${{ env.CACHE_NUMBER }}
 
       - name: Install Manager Environment
         shell: bash --noprofile --norc -eo pipefail {0}
@@ -128,9 +135,6 @@ jobs:
         if: inputs.env_manager != 'base'
         run: |
           pip install torch --index-url https://download.pytorch.org/whl/nightly/xpu
-          git clone --recursive https://github.com/pytorch/pytorch
-          cd pytorch
-          git checkout ${{ inputs.pytorch_ref }}
 
       - name: Install pass_rate dependencies
         run: |

From 32e03760177200e218dbd99e6ac5f611348c0042 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Tue, 15 Oct 2024 01:54:19 +0200
Subject: [PATCH 071/105] Fix pytorch install, r2

---
 .github/workflows/build-test-reusable.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 343897388b..e3c1bd5c16 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -80,6 +80,7 @@ jobs:
 
       - name: Clone PyTorch
         if: inputs.env_manager != 'base'
+        shell: bash --noprofile --norc -eo pipefail {0}
         run: |
           git clone --recursive https://github.com/pytorch/pytorch
           cd pytorch

From 2a5ca752f0512d7cbab6b4f1e1c3bf0f350d2552 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Tue, 15 Oct 2024 18:20:18 +0200
Subject: [PATCH 072/105] Fix transformer dependencies

---
 .github/workflows/build-test-reusable.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index e3c1bd5c16..6f9477a506 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -211,6 +211,11 @@ jobs:
           ref: ${{ env.TRANSFORMERS_VERSION }}
           try-tag-prefix: v
 
+      - name: Install transformers dependencies
+        if: (inputs.env_manager == 'base') && (steps.pip-cache.outputs.status == 'miss')
+        run: |
+          pip install pyyaml pandas scipy numpy psutil pyre_extensions torchrec
+
       - name: Install transformers
         if: (inputs.env_manager != 'base') && (steps.conda-cache.outputs.status == 'miss')
         run: |

From 804c031892c30a26d4a82e1f2c6161bc19331737 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Wed, 16 Oct 2024 09:15:41 +0200
Subject: [PATCH 073/105] Increase a cache number

---
 .github/workflows/build-test-reusable.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 6f9477a506..51a79cf0a7 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -113,7 +113,7 @@ jobs:
         uses: ./.github/actions/load
         env:
           # Increase this value to reset cache
-          CACHE_NUMBER: 1
+          CACHE_NUMBER: 2
         with:
           path: $HOME/.cache/pip
           key: pip-${{ inputs.python_version }}-${{ hashFiles('python/pyproject.toml', 'python/setup.py') }}-${{ env.CACHE_NUMBER }}

From da86e88dbf351c09a9e03b9e064835f9d4c66d88 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Wed, 16 Oct 2024 13:22:06 +0200
Subject: [PATCH 074/105] Fix psutil install, r1

---
 .github/workflows/build-test-reusable.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 51a79cf0a7..9e770e9feb 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -212,8 +212,9 @@ jobs:
           try-tag-prefix: v
 
       - name: Install transformers dependencies
-        if: (inputs.env_manager == 'base') && (steps.pip-cache.outputs.status == 'miss')
+        if: inputs.env_manager == 'base'
         run: |
+          #FIXME: somehow caching dows not work for psutil
           pip install pyyaml pandas scipy numpy psutil pyre_extensions torchrec
 
       - name: Install transformers

From 9d234d1623e8c7333611bc09ac98548c4811f0a5 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Wed, 16 Oct 2024 18:12:45 +0200
Subject: [PATCH 075/105] Fix psutil install, r2

---
 .github/workflows/build-test-reusable.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 9e770e9feb..c9eb41a57d 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -214,7 +214,6 @@ jobs:
       - name: Install transformers dependencies
         if: inputs.env_manager == 'base'
         run: |
-          #FIXME: somehow caching dows not work for psutil
           pip install pyyaml pandas scipy numpy psutil pyre_extensions torchrec
 
       - name: Install transformers

From 1da4d446c3932a97c7b2255b60c86d932fd88d6b Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Wed, 16 Oct 2024 19:26:57 +0200
Subject: [PATCH 076/105] Fix pytorch checkout

---
 .github/workflows/build-test-reusable.yml | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index c9eb41a57d..3111eeb463 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -78,16 +78,21 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Clone PyTorch
-        if: inputs.env_manager != 'base'
+      - name: Get pinned PyTorch ref
         shell: bash --noprofile --norc -eo pipefail {0}
+        if: inputs.env_manager != 'base' && inputs.ref == ''
         run: |
-          git clone --recursive https://github.com/pytorch/pytorch
-          cd pytorch
-          git checkout ${{ inputs.pytorch_ref }}
+          PYTORCH_REF="$(<.github/pins/pytorch-upstream.txt)"
+          echo "PYTORCH_REF=$PYTORCH_REF" | tee -a "$GITHUB_ENV"
 
-      - name: Load conda cache
+      - name: Clone PyTorch repository
         if: inputs.env_manager != 'base'
+        uses: actions/checkout@v4
+        with:
+          repository: pytorch/pytorch
+          ref: ${{ inputs.pytorch_ref || env.PYTORCH_REF }}
+
+      - name: Load conda cache
         id: conda-cache
         uses: ./.github/actions/load
         env:

From e18bea833f23e8fdcfd4ae0cb47da9eb1854caaa Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Wed, 16 Oct 2024 19:30:05 +0200
Subject: [PATCH 077/105] Fix pytorch checkout, r1

---
 .github/workflows/build-test-reusable.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 3111eeb463..0ea093adaa 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -91,6 +91,7 @@ jobs:
         with:
           repository: pytorch/pytorch
           ref: ${{ inputs.pytorch_ref || env.PYTORCH_REF }}
+          path: pytorch
 
       - name: Load conda cache
         id: conda-cache

From df03c337a7d93e0a67afd19a999e26ed8cd9b945 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Thu, 17 Oct 2024 18:47:09 +0200
Subject: [PATCH 078/105] Add a standalone e2e test, r1

---
 .github/workflows/build-test-reusable.yml | 55 ++++-------------------
 scripts/inductor_xpu_test.sh              |  2 +-
 2 files changed, 9 insertions(+), 48 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 0ea093adaa..1f8dbd1ebb 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -78,21 +78,6 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Get pinned PyTorch ref
-        shell: bash --noprofile --norc -eo pipefail {0}
-        if: inputs.env_manager != 'base' && inputs.ref == ''
-        run: |
-          PYTORCH_REF="$(<.github/pins/pytorch-upstream.txt)"
-          echo "PYTORCH_REF=$PYTORCH_REF" | tee -a "$GITHUB_ENV"
-
-      - name: Clone PyTorch repository
-        if: inputs.env_manager != 'base'
-        uses: actions/checkout@v4
-        with:
-          repository: pytorch/pytorch
-          ref: ${{ inputs.pytorch_ref || env.PYTORCH_REF }}
-          path: pytorch
-
       - name: Load conda cache
         id: conda-cache
         uses: ./.github/actions/load
@@ -100,7 +85,7 @@ jobs:
           CACHE_NUMBER: 6
         with:
           path: $HOME/miniforge3/envs/triton
-          key: conda-${{ inputs.env_manager }}-py${{ matrix.python }}-${{ hashFiles('scripts/triton.yml', 'python/pyproject.toml', 'python/setup.py', 'pytorch/.ci/docker/ci_commit_pins/huggingface.txt') }}-${{ env.CACHE_NUMBER }}
+          key: conda-${{ inputs.env_manager }}-py${{ matrix.python }}-${{ hashFiles('scripts/triton.yml', 'python/pyproject.toml', 'python/setup.py') }}-${{ env.CACHE_NUMBER }}
 
       - name: Install Manager Environment
         shell: bash --noprofile --norc -eo pipefail {0}
@@ -143,10 +128,6 @@ jobs:
         run: |
           pip install torch --index-url https://download.pytorch.org/whl/nightly/xpu
 
-      - name: Install pass_rate dependencies
-        run: |
-          pip install defusedxml
-
       - name: Setup Triton
         if: inputs.env_manager == 'base'
         uses: ./.github/actions/setup-triton
@@ -161,6 +142,12 @@ jobs:
           pip install pybind11
           pip install --no-build-isolation -e '.[build,tests,tutorials]'
 
+      - name: Run E2E test
+        run: |
+          #FIXME move back test when ready
+          scripts/inductor_xpu_test.sh huggingface float32 inference accuracy xpu 0 static 1 0 AlbertForMaskedLM
+          exit 1
+
       - name: Create test-triton command line
         run: |
           if [[ -n "${{ inputs.skip_list }}" ]]; then
@@ -217,33 +204,6 @@ jobs:
           ref: ${{ env.TRANSFORMERS_VERSION }}
           try-tag-prefix: v
 
-      - name: Install transformers dependencies
-        if: inputs.env_manager == 'base'
-        run: |
-          pip install pyyaml pandas scipy numpy psutil pyre_extensions torchrec
-
-      - name: Install transformers
-        if: (inputs.env_manager != 'base') && (steps.conda-cache.outputs.status == 'miss')
-        run: |
-          git clone --recursive https://github.com/huggingface/transformers
-          cd transformers
-          git checkout ${{ env.TRANSFORMERS_VERSION }}
-          python setup.py bdist_wheel
-          pip install dist/*.whl
-          python -c "import transformers; print(transformers.__version__)"
-          pip install pyyaml pandas scipy numpy psutil pyre_extensions torchrec
-
-      - name: Run E2E test
-        run: |
-          # Set WORKSPACE for inductor_xpu_test.sh to make sure it creates "inductor_log" outside of pytorch cloned directory
-          export WORKSPACE=$GITHUB_WORKSPACE
-          cd pytorch
-          # TODO: Find the fastest Hugging Face model
-          $GITHUB_WORKSPACE/scripts/inductor_xpu_test.sh huggingface float32 inference accuracy xpu 0 static 1 0 AlbertForMaskedLM
-          # The script above always returns 0, so we need an additional check to see if the accuracy test passed
-          cat $WORKSPACE/inductor_log/*/*/*.csv
-          grep AlbertForMaskedLM $WORKSPACE/inductor_log/*/*/*.csv | grep -q ,pass,
-
       - name: Save pip cache
         if: (inputs.env_manager == 'base') && (steps.pip-cache.outputs.status == 'miss')
         uses: ./.github/actions/save
@@ -260,6 +220,7 @@ jobs:
 
       - name: Pass rate
         run: |
+          pip install defusedxml
           source ./scripts/capture-hw-details.sh
           python3 scripts/pass_rate.py --reports reports ${{ env.SKIPLIST }}
           python3 scripts/pass_rate.py --reports reports --json ${{ env.SKIPLIST }} > pass_rate.json
diff --git a/scripts/inductor_xpu_test.sh b/scripts/inductor_xpu_test.sh
index 7373f6506c..4f1652afc3 100755
--- a/scripts/inductor_xpu_test.sh
+++ b/scripts/inductor_xpu_test.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 # This script work for xpu / cuda device inductor tests
-
+set -xv
 SUITE=${1:-huggingface}     # huggingface / torchbench / timm_models
 DT=${2:-float32}            # float32 / float16 / amp_bf16 / amp_fp16
 MODE=${3:-inference}        # inference / training

From 43e70d91bd6b435716d36229a27daf5360693d93 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Thu, 17 Oct 2024 19:16:16 +0200
Subject: [PATCH 079/105] Add a standalone e2e test, r2

---
 .github/workflows/build-test-reusable.yml |  3 ++-
 scripts/test-inductor.sh                  | 16 ++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)
 create mode 100755 scripts/test-inductor.sh

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 1f8dbd1ebb..cc4abc7264 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -145,7 +145,7 @@ jobs:
       - name: Run E2E test
         run: |
           #FIXME move back test when ready
-          scripts/inductor_xpu_test.sh huggingface float32 inference accuracy xpu 0 static 1 0 AlbertForMaskedLM
+          scripts/test-inductor.sh
           exit 1
 
       - name: Create test-triton command line
@@ -190,6 +190,7 @@ jobs:
           ${{ env.TRITON_TEST_CMD }} --instrumentation --skip-pip-install
 
       - name: Get transformers version
+        if: inputs.env_manager == 'base'
         run: |
           cd pytorch
           TRANSFORMERS_VERSION="$(<.ci/docker/ci_commit_pins/huggingface.txt)"
diff --git a/scripts/test-inductor.sh b/scripts/test-inductor.sh
new file mode 100755
index 0000000000..c2cbe1ebdc
--- /dev/null
+++ b/scripts/test-inductor.sh
@@ -0,0 +1,16 @@
+#/bin/sh
+
+test -d pytorch || {
+  git clone https://github.com/pytorch/pytorch
+  cd pytorch
+  TRANSFORMERS_VERSION=$(.ci/docker/ci_commit_pins/huggingface.txt)
+  pip install transformers=$TRANSFORMERS_VERSION
+  python -c "import transformers; print(transformers.__version__)"
+}
+
+pip install pyyaml pandas scipy numpy psutil pyre_extensions torchrec
+
+ZE_AFFINITY_MASK=0 python pytorch/benchmarks/dynamo/huggingface.py --accuracy --float32 -dxpu -n10 --no-skip --dashboard --inference --freezing --total-partitions 1 --partition-id 0 --only AlbertForMaskedLM --backend=inductor --timeout=4800 --output=inductor_log.csv
+
+cat inductor_log.csv
+grep AlbertForMaskedLM inductor_log.csv | grep -q ,pass,

From d30d33c84ffeefdbc05ce2232d74370f70585ce3 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Thu, 17 Oct 2024 19:25:29 +0200
Subject: [PATCH 080/105] Add a standalone e2e test, r3

---
 .github/workflows/build-test-reusable.yml | 1 -
 scripts/test-inductor.sh                  | 7 ++++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index cc4abc7264..351fe46faf 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -146,7 +146,6 @@ jobs:
         run: |
           #FIXME move back test when ready
           scripts/test-inductor.sh
-          exit 1
 
       - name: Create test-triton command line
         run: |
diff --git a/scripts/test-inductor.sh b/scripts/test-inductor.sh
index c2cbe1ebdc..72d68cbfdb 100755
--- a/scripts/test-inductor.sh
+++ b/scripts/test-inductor.sh
@@ -1,4 +1,7 @@
-#/bin/sh
+#!/bin/sh
+
+set -e
+set -vx
 
 test -d pytorch || {
   git clone https://github.com/pytorch/pytorch
@@ -12,5 +15,7 @@ pip install pyyaml pandas scipy numpy psutil pyre_extensions torchrec
 
 ZE_AFFINITY_MASK=0 python pytorch/benchmarks/dynamo/huggingface.py --accuracy --float32 -dxpu -n10 --no-skip --dashboard --inference --freezing --total-partitions 1 --partition-id 0 --only AlbertForMaskedLM --backend=inductor --timeout=4800 --output=inductor_log.csv
 
+find .
+
 cat inductor_log.csv
 grep AlbertForMaskedLM inductor_log.csv | grep -q ,pass,

From ae9a9d4aa8693c6056a77ec2a3d481a6cbeaf26f Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Thu, 17 Oct 2024 19:33:09 +0200
Subject: [PATCH 081/105] Add a standalone e2e test, r4

---
 scripts/test-inductor.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/test-inductor.sh b/scripts/test-inductor.sh
index 72d68cbfdb..59c8120f0e 100755
--- a/scripts/test-inductor.sh
+++ b/scripts/test-inductor.sh
@@ -6,7 +6,7 @@ set -vx
 test -d pytorch || {
   git clone https://github.com/pytorch/pytorch
   cd pytorch
-  TRANSFORMERS_VERSION=$(.ci/docker/ci_commit_pins/huggingface.txt)
+  TRANSFORMERS_VERSION=$(cat .ci/docker/ci_commit_pins/huggingface.txt)
   pip install transformers=$TRANSFORMERS_VERSION
   python -c "import transformers; print(transformers.__version__)"
 }

From 7858466e8855022767792d2c665842906633e49e Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Thu, 17 Oct 2024 19:39:33 +0200
Subject: [PATCH 082/105] Add a standalone e2e test, r5

---
 .github/workflows/build-test-reusable.yml | 10 ++++-----
 scripts/test-inductor.sh                  | 21 ------------------
 scripts/test-triton.sh                    | 27 ++++++++++++++++++++++-
 3 files changed, 31 insertions(+), 27 deletions(-)
 delete mode 100755 scripts/test-inductor.sh

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 351fe46faf..8455e21429 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -142,11 +142,6 @@ jobs:
           pip install pybind11
           pip install --no-build-isolation -e '.[build,tests,tutorials]'
 
-      - name: Run E2E test
-        run: |
-          #FIXME move back test when ready
-          scripts/test-inductor.sh
-
       - name: Create test-triton command line
         run: |
           if [[ -n "${{ inputs.skip_list }}" ]]; then
@@ -168,6 +163,11 @@ jobs:
             echo TRITON_TEST_CMD="bash -v -x scripts/test-triton.sh --warning-reports --skip-pytorch-install --reports-dir $GITHUB_WORKSPACE/reports ${{ inputs.ignore_errors && '--ignore-errors' || '' }} $skiplist"
           } | tee -a $GITHUB_ENV
 
+      - name: Run inductor test
+        run: |
+          #FIXME move back test when ready
+          ${{ env.TRITON_TEST_CMD }} --inductor #--skip-pip-install
+
       - name: Run unit tests
         run: |
           ${{ env.TRITON_TEST_CMD }} --unit
diff --git a/scripts/test-inductor.sh b/scripts/test-inductor.sh
deleted file mode 100755
index 59c8120f0e..0000000000
--- a/scripts/test-inductor.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/sh
-
-set -e
-set -vx
-
-test -d pytorch || {
-  git clone https://github.com/pytorch/pytorch
-  cd pytorch
-  TRANSFORMERS_VERSION=$(cat .ci/docker/ci_commit_pins/huggingface.txt)
-  pip install transformers=$TRANSFORMERS_VERSION
-  python -c "import transformers; print(transformers.__version__)"
-}
-
-pip install pyyaml pandas scipy numpy psutil pyre_extensions torchrec
-
-ZE_AFFINITY_MASK=0 python pytorch/benchmarks/dynamo/huggingface.py --accuracy --float32 -dxpu -n10 --no-skip --dashboard --inference --freezing --total-partitions 1 --partition-id 0 --only AlbertForMaskedLM --backend=inductor --timeout=4800 --output=inductor_log.csv
-
-find .
-
-cat inductor_log.csv
-grep AlbertForMaskedLM inductor_log.csv | grep -q ,pass,
diff --git a/scripts/test-triton.sh b/scripts/test-triton.sh
index 8df7009dcd..da573ba31a 100755
--- a/scripts/test-triton.sh
+++ b/scripts/test-triton.sh
@@ -19,6 +19,7 @@ TEST_BENCHMARK_SOFTMAX=false
 TEST_BENCHMARK_GEMM=false
 TEST_BENCHMARK_ATTENTION=false
 TEST_INSTRUMENTATION=false
+TEST_INDUCTOR=false
 VENV=false
 TRITON_TEST_REPORTS=false
 TRITON_TEST_WARNING_REPORTS=false
@@ -68,6 +69,10 @@ while [ -v 1 ]; do
       TEST_INSTRUMENTATION=true
       shift
       ;;
+    --inductor)
+      TEST_INDUCTOR=true
+      shift
+      ;;
     --venv)
       VENV=true
       shift
@@ -111,7 +116,7 @@ while [ -v 1 ]; do
 done
 
 # Only run interpreter test when $TEST_INTERPRETER is true
-if [ "$TEST_UNIT" = false ] && [ "$TEST_CORE" = false ] && [ "$TEST_INTERPRETER" = false ] && [ "$TEST_TUTORIAL" = false ] && [ "$TEST_MICRO_BENCHMARKS" = false ] && [ "$TEST_BENCHMARK_SOFTMAX" = false ] && [ "$TEST_BENCHMARK_GEMM" = false ] && [ "$TEST_BENCHMARK_ATTENTION" = false ] && [ "$TEST_INSTRUMENTATION" = false ]; then
+if [ "$TEST_UNIT" = false ] && [ "$TEST_CORE" = false ] && [ "$TEST_INTERPRETER" = false ] && [ "$TEST_TUTORIAL" = false ] && [ "$TEST_MICRO_BENCHMARKS" = false ] && [ "$TEST_BENCHMARK_SOFTMAX" = false ] && [ "$TEST_BENCHMARK_GEMM" = false ] && [ "$TEST_BENCHMARK_ATTENTION" = false ] && [ "$TEST_INSTRUMENTATION" = false ] && [ "$TEST_INDUCTOR" = false ]; then
   TEST_UNIT=true
   TEST_CORE=true
   TEST_TUTORIAL=true
@@ -307,6 +312,23 @@ run_instrumentation_tests() {
     pytest -vvv --device xpu instrumentation/test_gpuhello.py
 }
 
+run_inductor_tests() {
+  test -d pytorch || {
+    git clone https://github.com/pytorch/pytorch
+    cd pytorch
+    TRANSFORMERS_VERSION=$(cat .ci/docker/ci_commit_pins/huggingface.txt)
+    pip install transformers==$TRANSFORMERS_VERSION
+    python -c "import transformers; print(transformers.__version__)"
+  }
+
+  pip install pyyaml pandas scipy numpy psutil pyre_extensions torchrec
+
+  ZE_AFFINITY_MASK=0 python pytorch/benchmarks/dynamo/huggingface.py --accuracy --float32 -dxpu -n10 --no-skip --dashboard --inference --freezing --total-partitions 1 --partition-id 0 --only AlbertForMaskedLM --backend=inductor --timeout=4800 --output=inductor_log.csv
+
+  cat inductor_log.csv
+  grep AlbertForMaskedLM inductor_log.csv | grep -q ,pass,
+}
+
 test_triton() {
   if [ "$TEST_UNIT" = true ]; then
     run_unit_tests
@@ -336,6 +358,9 @@ test_triton() {
   if [ "$TEST_INSTRUMENTATION" == true ]; then
     run_instrumentation_tests
   fi
+  if [ "$TEST_INDUCTOR" == true ]; then
+    run_inductor_tests
+  fi
 }
 
 install_deps

From 59bdfe030e5b8b52150405d441e6794b219dd265 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Thu, 17 Oct 2024 19:48:13 +0200
Subject: [PATCH 083/105] Add a standalone e2e test, r6

---
 scripts/test-triton.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/scripts/test-triton.sh b/scripts/test-triton.sh
index da573ba31a..6fb1413e9a 100755
--- a/scripts/test-triton.sh
+++ b/scripts/test-triton.sh
@@ -316,8 +316,13 @@ run_inductor_tests() {
   test -d pytorch || {
     git clone https://github.com/pytorch/pytorch
     cd pytorch
+
     TRANSFORMERS_VERSION=$(cat .ci/docker/ci_commit_pins/huggingface.txt)
-    pip install transformers==$TRANSFORMERS_VERSION
+    git clone --recursive https://github.com/huggingface/transformers
+    cd transformers
+    git checkout $TRANSFORMERS_VERSION
+    python setup.py bdist_wheel
+    pip install dist/*.whl
     python -c "import transformers; print(transformers.__version__)"
   }
 

From 5ac822d0229adb849bf22e1baa9ad7baf3570ad7 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Thu, 17 Oct 2024 20:07:09 +0200
Subject: [PATCH 084/105] Add a standalone e2e test, r7

---
 .github/workflows/build-test-reusable.yml | 12 ++++++------
 scripts/test-triton.sh                    | 10 ++--------
 2 files changed, 8 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 8455e21429..9b8fb78ad5 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -134,13 +134,13 @@ jobs:
         with:
           build_llvm: ${{ inputs.build_llvm }}
 
-      - name: Setup Triton
+      - name: Setup Triton and Transformers
         if: inputs.env_manager != 'base'
-        run: |
-          export DEBUG=1
-          cd python
-          pip install pybind11
-          pip install --no-build-isolation -e '.[build,tests,tutorials]'
+        uses: ./.github/actions/install-wheels
+        with:
+          gh_token: ${{ secrets.GITHUB_TOKEN }}
+          install_cmd: conda run --no-capture-output -n triton pip install
+          python_version: ${{ matrix.python }}
 
       - name: Create test-triton command line
         run: |
diff --git a/scripts/test-triton.sh b/scripts/test-triton.sh
index 6fb1413e9a..934bf32c8c 100755
--- a/scripts/test-triton.sh
+++ b/scripts/test-triton.sh
@@ -315,15 +315,9 @@ run_instrumentation_tests() {
 run_inductor_tests() {
   test -d pytorch || {
     git clone https://github.com/pytorch/pytorch
+    rev=$(cat .github/pins/pytorch-upstream.txt)
     cd pytorch
-
-    TRANSFORMERS_VERSION=$(cat .ci/docker/ci_commit_pins/huggingface.txt)
-    git clone --recursive https://github.com/huggingface/transformers
-    cd transformers
-    git checkout $TRANSFORMERS_VERSION
-    python setup.py bdist_wheel
-    pip install dist/*.whl
-    python -c "import transformers; print(transformers.__version__)"
+    git checkout $rev
   }
 
   pip install pyyaml pandas scipy numpy psutil pyre_extensions torchrec

From 19dc278788979a436e0fa70be70e152e21ad83df Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Thu, 17 Oct 2024 20:12:31 +0200
Subject: [PATCH 085/105] Add a standalone e2e test, r8

---
 .github/workflows/build-test-reusable.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 9b8fb78ad5..4228148c73 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -140,7 +140,7 @@ jobs:
         with:
           gh_token: ${{ secrets.GITHUB_TOKEN }}
           install_cmd: conda run --no-capture-output -n triton pip install
-          python_version: ${{ matrix.python }}
+          python_version: ${{ inputs.python_version }}
 
       - name: Create test-triton command line
         run: |

From a06d8435c51af6b6ee68db0daf31812e69154cb9 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Thu, 17 Oct 2024 20:16:55 +0200
Subject: [PATCH 086/105] Add a standalone e2e test, r9

---
 .github/workflows/build-test-reusable.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 4228148c73..aedf98453b 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -139,7 +139,7 @@ jobs:
         uses: ./.github/actions/install-wheels
         with:
           gh_token: ${{ secrets.GITHUB_TOKEN }}
-          install_cmd: conda run --no-capture-output -n triton pip install
+          install_cmd: ${{ env.HOME }}/miniforge3/bin/conda run --no-capture-output -n triton pip install
           python_version: ${{ inputs.python_version }}
 
       - name: Create test-triton command line

From ccc9a737076a475df477e595a750fcf68ec35c5a Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Thu, 17 Oct 2024 21:55:13 +0200
Subject: [PATCH 087/105] Fix conda path

---
 .github/workflows/build-test-reusable.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index aedf98453b..7425e5916c 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -91,6 +91,7 @@ jobs:
         shell: bash --noprofile --norc -eo pipefail {0}
         run: |
           scripts/install-${{ inputs.env_manager }}.sh --python-version ${{ inputs.python_version }}
+          echo $HOME/miniforge3/bin >>$GITHUB_PATH
 
       - name: Print inputs
         run: |
@@ -139,7 +140,7 @@ jobs:
         uses: ./.github/actions/install-wheels
         with:
           gh_token: ${{ secrets.GITHUB_TOKEN }}
-          install_cmd: ${{ env.HOME }}/miniforge3/bin/conda run --no-capture-output -n triton pip install
+          install_cmd: conda run --no-capture-output -n triton pip install
           python_version: ${{ inputs.python_version }}
 
       - name: Create test-triton command line

From ebf45ec719079bf3e7dce84b63021565ee6e4d08 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Thu, 17 Oct 2024 22:07:28 +0200
Subject: [PATCH 088/105] Fix pytorch path

---
 .github/workflows/build-test-reusable.yml |   4 +
 scripts/test-triton.sh                    | 562 ++++++++++------------
 2 files changed, 267 insertions(+), 299 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 7425e5916c..45bbcec9a5 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -87,6 +87,10 @@ jobs:
           path: $HOME/miniforge3/envs/triton
           key: conda-${{ inputs.env_manager }}-py${{ matrix.python }}-${{ hashFiles('scripts/triton.yml', 'python/pyproject.toml', 'python/setup.py') }}-${{ env.CACHE_NUMBER }}
 
+      - name: Fail
+        if: inputs.env_manager == 'base'
+        run: exit 1
+
       - name: Install Manager Environment
         shell: bash --noprofile --norc -eo pipefail {0}
         run: |
diff --git a/scripts/test-triton.sh b/scripts/test-triton.sh
index 934bf32c8c..ec9441a3ac 100755
--- a/scripts/test-triton.sh
+++ b/scripts/test-triton.sh
@@ -2,323 +2,287 @@
 
 set -euo pipefail
 
-err() {
-    echo $@
-    exit 1
-}
+err() { echo $@ exit 1 }
 
 export PIP_DISABLE_PIP_VERSION_CHECK=1
 
 # Select which tests to run.
-TEST_UNIT=false
-TEST_CORE=false
-TEST_INTERPRETER=false
-TEST_TUTORIAL=false
-TEST_MICRO_BENCHMARKS=false
-TEST_BENCHMARK_SOFTMAX=false
-TEST_BENCHMARK_GEMM=false
-TEST_BENCHMARK_ATTENTION=false
-TEST_INSTRUMENTATION=false
-TEST_INDUCTOR=false
-VENV=false
-TRITON_TEST_REPORTS=false
-TRITON_TEST_WARNING_REPORTS=false
-TRITON_TEST_IGNORE_ERRORS=false
-SKIP_PIP=false
-SKIP_PYTORCH=false
-TEST_UNSKIP=false
-while [ -v 1 ]; do
-  case "$1" in
-    --unskip)
-      TEST_UNSKIP=true
-      shift
-      ;;
-    --unit)
-      TEST_UNIT=true
-      shift
-      ;;
-    --core)
-      TEST_CORE=true
-      shift
-      ;;
-    --interpreter)
-      TEST_INTERPRETER=true
-      shift
-      ;;
-    --tutorial)
-      TEST_TUTORIAL=true
-      shift
-      ;;
-    --microbench)
-      TEST_MICRO_BENCHMARKS=true
-      shift
-      ;;
-    --softmax)
-      TEST_BENCHMARK_SOFTMAX=true
-      shift
-      ;;
-    --gemm)
-      TEST_BENCHMARK_GEMM=true
-      shift
-      ;;
-    --attention)
-      TEST_BENCHMARK_ATTENTION=true
-      shift
-      ;;
-    --instrumentation)
-      TEST_INSTRUMENTATION=true
-      shift
-      ;;
-    --inductor)
-      TEST_INDUCTOR=true
-      shift
-      ;;
-    --venv)
-      VENV=true
-      shift
-      ;;
-    --skip-pip-install)
-      SKIP_PIP=true
-      shift
-      ;;
-    --skip-pytorch-install)
-      SKIP_PYTORCH=true
-      shift
-      ;;
-    --reports)
-      TRITON_TEST_REPORTS=true
-      shift
-      ;;
-    --reports-dir)
-      TRITON_TEST_REPORTS=true
-      TRITON_TEST_REPORTS_DIR="$2"
-      shift 2
-      ;;
-    --warning-reports)
-      TRITON_TEST_WARNING_REPORTS=true
-      shift
-      ;;
-    --ignore-errors)
-      TRITON_TEST_IGNORE_ERRORS=true
-      shift
-      ;;
-    --skip-list)
-      TRITON_TEST_SKIPLIST_DIR="$2"
-      shift 2
-      ;;
-    --help)
-      err "Example usage: ./test-triton.sh [--core | --tutorial | --unit | --microbench | --softmax | --gemm | --attention | --venv | --skip-pip-install | --skip-pytorch-install | --reports | --reports-dir DIR | --warning-reports | --ignore-errors | --skip-list SKIPLIST"
-      ;;
-    *)
-      err "Unknown argument: $1."
-      ;;
-  esac
-done
+TEST_UNIT=false TEST_CORE=false TEST_INTERPRETER=false TEST_TUTORIAL=false
+TEST_MICRO_BENCHMARKS=false TEST_BENCHMARK_SOFTMAX=false
+TEST_BENCHMARK_GEMM=false TEST_BENCHMARK_ATTENTION=false
+TEST_INSTRUMENTATION=false TEST_INDUCTOR=false VENV=false
+TRITON_TEST_REPORTS=false TRITON_TEST_WARNING_REPORTS=false
+TRITON_TEST_IGNORE_ERRORS=false SKIP_PIP=false SKIP_PYTORCH=false
+TEST_UNSKIP=false while [ -v 1 ]; do case "$1" in --unskip) TEST_UNSKIP=true
+shift ;; --unit) TEST_UNIT=true shift ;; --core) TEST_CORE=true shift ;;
+--interpreter) TEST_INTERPRETER=true shift ;; --tutorial) TEST_TUTORIAL=true
+        shift ;; --microbench) TEST_MICRO_BENCHMARKS=true shift ;; --softmax)
+        TEST_BENCHMARK_SOFTMAX=true shift ;; --gemm) TEST_BENCHMARK_GEMM=true
+        shift ;; --attention) TEST_BENCHMARK_ATTENTION=true shift ;;
+        --instrumentation) TEST_INSTRUMENTATION=true shift ;; --inductor)
+                TEST_INDUCTOR=true shift ;; --venv) VENV=true shift ;;
+                --skip-pip-install) SKIP_PIP=true shift ;;
+                --skip-pytorch-install) SKIP_PYTORCH=true shift ;; --reports)
+                        TRITON_TEST_REPORTS=true shift ;; --reports-dir)
+                        TRITON_TEST_REPORTS=true TRITON_TEST_REPORTS_DIR="$2"
+                        shift 2 ;; --warning-reports)
+                        TRITON_TEST_WARNING_REPORTS=true shift ;;
+                --ignore-errors) TRITON_TEST_IGNORE_ERRORS=true shift ;;
+                --skip-list) TRITON_TEST_SKIPLIST_DIR="$2" shift 2 ;; --help)
+                        err "Example usage: ./test-triton.sh [--core |
+                                --tutorial | --unit | --microbench | --softmax
+                                                        | --gemm | --attention
+                                                        | --venv |
+                                                                --skip-pip-install
+                                                                                                                        |
+                                                                                                                                --skip-pytorch-install
+                                                                                                                                                                                                                                                        |
+                                                                                                                                                                                                                                                                --reports
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                --reports-dir
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        DIR | --warning-reports | --ignore-errors | --skip-list SKIPLIST" ;; *) err "Unknown argument: $1." ;; esac done
 
 # Only run interpreter test when $TEST_INTERPRETER is true
-if [ "$TEST_UNIT" = false ] && [ "$TEST_CORE" = false ] && [ "$TEST_INTERPRETER" = false ] && [ "$TEST_TUTORIAL" = false ] && [ "$TEST_MICRO_BENCHMARKS" = false ] && [ "$TEST_BENCHMARK_SOFTMAX" = false ] && [ "$TEST_BENCHMARK_GEMM" = false ] && [ "$TEST_BENCHMARK_ATTENTION" = false ] && [ "$TEST_INSTRUMENTATION" = false ] && [ "$TEST_INDUCTOR" = false ]; then
-  TEST_UNIT=true
-  TEST_CORE=true
-  TEST_TUTORIAL=true
-  TEST_MICRO_BENCHMARKS=true
-fi
-
-if [ "$VENV" = true ]; then
-  source .venv/bin/activate
-fi
-
-
-TRITON_PROJ="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && cd .. && pwd )"
-SCRIPTS_DIR="$TRITON_PROJ/scripts"
-source "$SCRIPTS_DIR/pytest-utils.sh"
-
-if [ "$TRITON_TEST_REPORTS" == true ]; then
-    capture_runtime_env
-fi
-
-install_deps() {
-  if [ "$SKIP_PIP" = true ]; then
-    echo "**** Skipping installation of pip dependencies ****"
-  else
-    echo "**** Installing pip dependencies ****"
-    python -m pip install -r "$SCRIPTS_DIR/requirements-test.txt"
-
-    if [ "$TRITON_TEST_WARNING_REPORTS" == true ]; then
-      python -m pip install git+https://github.com/kwasd/pytest-capturewarnings-ng@v1.2.0
-    fi
-  fi
-
-  if [ "$SKIP_PYTORCH" = true ]; then
-    echo "**** Skipping installation of pytorch ****"
-  else
-    echo "**** Installing pytorch ****"
-    if [ "$TEST_BENCHMARK_SOFTMAX" = true ] || [ "$TEST_BENCHMARK_GEMM" = true ] || [ "$TEST_BENCHMARK_ATTENTION" = true ]; then
-      $SCRIPTS_DIR/compile-pytorch-ipex.sh $([ $VENV = true ] && echo "--venv")
-    else
-      $SCRIPTS_DIR/install-pytorch.sh $([ $VENV = true ] && echo "--venv")
-    fi
-  fi
-}
-
-run_unit_tests() {
-  echo "***************************************************"
-  echo "******      Running Triton CXX unittests     ******"
-  echo "***************************************************"
-  cd $TRITON_PROJ/python/build/cmake* || err "****** ERROR: Build Triton first ******"
-  ctest .
-
-  echo "***************************************************"
-  echo "******       Running Triton LIT tests        ******"
-  echo "***************************************************"
-  cd $TRITON_PROJ/python/build/cmake*/test
-  lit -v .
-}
-
-run_core_tests() {
-  echo "***************************************************"
-  echo "******      Running Triton Core tests        ******"
-  echo "***************************************************"
-  cd $TRITON_PROJ/python/test/unit
-  ensure_spirv_dis
-
-  TRITON_DISABLE_LINE_INFO=1 TRITON_TEST_SUITE=language \
-    pytest -vvv -n ${PYTEST_MAX_PROCESSES:-8} --device xpu language/ --ignore=language/test_line_info.py --ignore=language/test_subprocess.py
-
-  TRITON_DISABLE_LINE_INFO=1 TRITON_TEST_SUITE=subprocess \
-    pytest -vvv -n ${PYTEST_MAX_PROCESSES:-8} --device xpu language/test_subprocess.py
+if [ "$TEST_UNIT" = false ] && [ "$TEST_CORE" = false ] && [
+        "$TEST_INTERPRETER" = false ] && [ "$TEST_TUTORIAL" = false ] && [
+        "$TEST_MICRO_BENCHMARKS" = false ] && [ "$TEST_BENCHMARK_SOFTMAX" =
+        false ] && [ "$TEST_BENCHMARK_GEMM" = false ] && [
+        "$TEST_BENCHMARK_ATTENTION" = false ] && [ "$TEST_INSTRUMENTATION" =
+        false ] && [ "$TEST_INDUCTOR" = false ]; then TEST_UNIT=true
+        TEST_CORE=true TEST_TUTORIAL=true TEST_MICRO_BENCHMARKS=true fi
+
+        if [ "$VENV" = true ]; then source .venv/bin/activate fi
+
+
+        TRITON_PROJ="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &>
+        /dev/null && cd .. && pwd )" SCRIPTS_DIR="$TRITON_PROJ/scripts" source
+        "$SCRIPTS_DIR/pytest-utils.sh"
+
+        if [ "$TRITON_TEST_REPORTS" == true ]; then capture_runtime_env fi
+
+        install_deps() { if [ "$SKIP_PIP" = true ]; then echo "**** Skipping
+                installation of pip dependencies ****" else echo "****
+                Installing pip dependencies ****" python -m pip install -r
+                "$SCRIPTS_DIR/requirements-test.txt"
+
+                if [ "$TRITON_TEST_WARNING_REPORTS" == true ]; then python -m
+                        pip install
+                        git+https://github.com/kwasd/pytest-capturewarnings-ng@v1.2.0
+                        fi fi
+
+                        if [ "$SKIP_PYTORCH" = true ]; then echo "**** Skipping
+                                installation of pytorch ****" else echo "****
+                                Installing pytorch ****" if [
+                                "$TEST_BENCHMARK_SOFTMAX" = true ] || [
+                                "$TEST_BENCHMARK_GEMM" = true ] || [
+                                "$TEST_BENCHMARK_ATTENTION" = true ]; then
+                                $SCRIPTS_DIR/compile-pytorch-ipex.sh $([ $VENV
+                                = true ] && echo "--venv") else
+                                $SCRIPTS_DIR/install-pytorch.sh $([ $VENV =
+                                true ] && echo "--venv") fi fi }
+
+                                run_unit_tests() { echo
+                                        "***************************************************"
+                                        echo "******      Running Triton CXX
+                                        unittests     ******" echo
+                                        "***************************************************"
+                                        cd $TRITON_PROJ/python/build/cmake* ||
+                                                err "****** ERROR: Build Triton
+                                                                                        first
+                                                                                        ******"
+                                                                                        ctest
+                                                                                        .
+
+                                                                                        echo
+                                                                                        "***************************************************"
+                                                                                        echo
+                                                                                        "******
+                                                                                        Running
+                                                                                        Triton
+                                                                                        LIT
+                                                                                        tests
+                                                                                        ******"
+                                                                                        echo
+                                                                                        "***************************************************"
+                                                                                        cd
+                                                                                        $TRITON_PROJ/python/build/cmake*/test
+                                                                                        lit
+                                                                                        -v
+                                                                                        .
+                                                                                }
+
+                                                                                run_core_tests()
+                                                                                {
+                                                                                        echo
+                                                                                        "***************************************************"
+                                                                                        echo
+                                                                                        "******
+                                                                                        Running
+                                                                                        Triton
+                                                                                        Core
+                                                                                        tests
+                                                                                        ******"
+                                                                                        echo
+                                                                                        "***************************************************"
+                                                                                        cd
+                                                                                        $TRITON_PROJ/python/test/unit
+                                                                                        ensure_spirv_dis
+
+                                                                                        TRITON_DISABLE_LINE_INFO=1
+                                                                                        TRITON_TEST_SUITE=language
+                                                                                        \
+                                                                                                pytest
+                                                                                                                                                                                        -vvv
+                                                                                                                                                                                        -n
+                                                                                                                                                                                        ${PYTEST_MAX_PROCESSES:-8}
+                                                                                                                                                                                        --device
+                                                                                                                                                                                        xpu
+                                                                                                                                                                                        language/
+                                                                                                                                                                                        --ignore=language/test_line_info.py
+                                                                                                                                                                                        --ignore=language/test_subprocess.py
+
+                                                                                                                                                                                        TRITON_DISABLE_LINE_INFO=1
+                                                                                                                                                                                        TRITON_TEST_SUITE=subprocess
+                                                                                                                                                                                        \
+                                                                                                                                                                                                pytest
+                                                                                                                                                                                                                                                                                                                                                                                        -vvv
+                                                                                                                                                                                                                                                                                                                                                                                        -n
+                                                                                                                                                                                                                                                                                                                                                                                        ${PYTEST_MAX_PROCESSES:-8}
+                                                                                                                                                                                                                                                                                                                                                                                        --device
+                                                                                                                                                                                                                                                                                                                                                                                        xpu
+                                                                                                                                                                                                                                                                                                                                                                                        language/test_subprocess.py
 
   # run runtime tests serially to avoid race condition with cache handling.
-  TRITON_DISABLE_LINE_INFO=1 TRITON_TEST_SUITE=runtime \
-    pytest --verbose --device xpu runtime/
+  TRITON_DISABLE_LINE_INFO=1 TRITON_TEST_SUITE=runtime \ pytest --verbose
+  --device xpu runtime/
 
   # run test_line_info.py separately with TRITON_DISABLE_LINE_INFO=0
-  TRITON_DISABLE_LINE_INFO=0 TRITON_TEST_SUITE=line_info \
-    pytest -k "not test_line_info_interpreter" --verbose --device xpu language/test_line_info.py
-}
-
-run_regression_tests() {
-  echo "***************************************************"
-  echo "******   Running Triton Regression tests     ******"
-  echo "***************************************************"
-  cd $TRITON_PROJ/python/test/regression
-
-  TRITON_DISABLE_LINE_INFO=1 TRITON_TEST_SUITE=regression \
-    pytest -vvv -s --device xpu . --ignore=test_performance.py
-}
-
-run_interpreter_tests() {
-  echo "***************************************************"
-  echo "******   Running Triton Interpreter tests    ******"
-  echo "***************************************************"
-  cd $TRITON_PROJ/python/test/unit
-
-  TRITON_INTERPRET=1 TRITON_TEST_SUITE=interpreter \
-    pytest -vvv -n ${PYTEST_MAX_PROCESSES:-16} -m interpreter language/test_core.py language/test_standard.py \
-    language/test_random.py language/test_line_info.py --device cpu
+  TRITON_DISABLE_LINE_INFO=0 TRITON_TEST_SUITE=line_info \ pytest -k "not
+  test_line_info_interpreter" --verbose --device xpu language/test_line_info.py
 }
 
-run_tutorial_tests() {
-  echo "***************************************************"
-  echo "**** Running Triton Tutorial tests           ******"
-  echo "***************************************************"
-  python -m pip install matplotlib pandas tabulate -q
-  cd $TRITON_PROJ/python/tutorials
-
-  run_tutorial_test "01-vector-add"
-  run_tutorial_test "02-fused-softmax"
-  run_tutorial_test "03-matrix-multiplication"
-  run_tutorial_test "04-low-memory-dropout"
-  run_tutorial_test "05-layer-norm"
-  run_tutorial_test "06-fused-attention"
-  run_tutorial_test "07-extern-functions"
-  run_tutorial_test "08-grouped-gemm"
-  run_tutorial_test "10-experimental-block-pointer"
-  run_tutorial_test "10i-experimental-block-pointer"
-}
-
-run_microbench_tests() {
-  echo "****************************************************"
-  echo "*****   Running Triton Micro Benchmark tests   *****"
-  echo "****************************************************"
-  USE_IPEX=0 python $TRITON_PROJ/benchmarks/micro_benchmarks/run_benchmarks.py
-}
-
-run_benchmark_softmax() {
-  echo "****************************************************"
-  echo "*****             Running Softmax              *****"
-  echo "****************************************************"
-  cd $TRITON_PROJ/benchmarks
-  python setup.py install
-  python $TRITON_PROJ/benchmarks/triton_kernels_benchmark/fused_softmax.py
-}
-
-run_benchmark_gemm() {
-  echo "****************************************************"
-  echo "*****              Running GEMM                *****"
-  echo "****************************************************"
-  cd $TRITON_PROJ/benchmarks
-  python setup.py install
-
-  echo "Default path:"
-  TRITON_INTEL_ADVANCED_PATH=0 \
-    TRITON_INTEL_ENABLE_ADDRESS_PAYLOAD_OPT=1 \
-    IGC_VISAOptions=" -enableBCR -nolocalra" \
-    IGC_DisableLoopUnroll=1 \
-    python $TRITON_PROJ/benchmarks/triton_kernels_benchmark/gemm_benchmark.py
-
-  echo "Advanced path:"
-  TRITON_INTEL_ADVANCED_PATH=1 \
-    TRITON_INTEL_ENABLE_ADDRESS_PAYLOAD_OPT=1 \
-    IGC_VISAOptions=" -enableBCR -nolocalra" \
-    IGC_DisableLoopUnroll=1 \
-    python $TRITON_PROJ/benchmarks/triton_kernels_benchmark/gemm_benchmark.py
-}
-
-run_benchmark_attention() {
-  echo "****************************************************"
-  echo "*****            Running ATTENTION             *****"
-  echo "****************************************************"
-  cd $TRITON_PROJ/benchmarks
-  python setup.py install
-
-  echo "Default path:"
-  TRITON_INTEL_ADVANCED_PATH=0 \
-    TRITON_INTEL_ENABLE_ADDRESS_PAYLOAD_OPT=1 \
-    IGC_VISAOptions=" -enableBCR" \
-    python $TRITON_PROJ/benchmarks/triton_kernels_benchmark/flash_attention_fwd_benchmark.py
-
-  echo "Advanced path:"
-  TRITON_INTEL_ADVANCED_PATH=1 \
-    TRITON_INTEL_ENABLE_ADDRESS_PAYLOAD_OPT=1 \
-    TRITON_INTEL_ENABLE_INSTR_SCHED=1 \
-    IGC_VISAOptions=" -enableBCR" \
-    python $TRITON_PROJ/benchmarks/triton_kernels_benchmark/flash_attention_fwd_benchmark.py
-}
-
-run_instrumentation_tests() {
-  # FIXME: the "instrumentation" test suite currently contains only one test, when all tests
-  # are skipped pytest reports an error. If the only test is the skip list, then we shouldn't
-  # run pytest at all. This must be changed when there is more than one instrumentation test.
-  if [[ $TEST_UNSKIP = false && -s $TRITON_TEST_SKIPLIST_DIR/instrumentation.txt ]]; then
-    return
-  fi
-
-  SHARED_LIB_DIR=$(ls -1d $TRITON_PROJ/python/build/*lib*/triton/_C) || SHARED_LIB_DIR=$(ls -1d $TRITON_PROJ/python/triton/_C) || err "Could not find $TRITON_PROJ/python/build/*lib*/triton/_C, build Triton first"
-
-  cd $TRITON_PROJ/python/test/unit
-
-  TRITON_TEST_SUITE=instrumentation \
-    TRITON_ALWAYS_COMPILE=1 TRITON_DISABLE_LINE_INFO=0 LLVM_PASS_PLUGIN_PATH=${SHARED_LIB_DIR}/libGPUHello.so \
-    pytest -vvv --device xpu instrumentation/test_gpuhello.py
-}
+run_regression_tests() { echo
+        "***************************************************" echo "******
+        Running Triton Regression tests     ******" echo
+        "***************************************************" cd
+        $TRITON_PROJ/python/test/regression
+
+        TRITON_DISABLE_LINE_INFO=1 TRITON_TEST_SUITE=regression \ pytest -vvv
+        -s --device xpu . --ignore=test_performance.py }
+
+        run_interpreter_tests() { echo
+                "***************************************************" echo
+                "******   Running Triton Interpreter tests    ******" echo
+                "***************************************************" cd
+                $TRITON_PROJ/python/test/unit
+
+                TRITON_INTERPRET=1 TRITON_TEST_SUITE=interpreter \ pytest -vvv
+                -n ${PYTEST_MAX_PROCESSES:-16} -m interpreter
+                language/test_core.py language/test_standard.py \
+                        language/test_random.py language/test_line_info.py
+                                        --device cpu }
+
+                                        run_tutorial_tests() { echo
+                                                "***************************************************"
+                                                echo "**** Running Triton
+                                                Tutorial tests
+                                                ******" echo
+                                                "***************************************************"
+                                                python -m pip install
+                                                matplotlib pandas tabulate -q
+                                                cd
+                                                $TRITON_PROJ/python/tutorials
+
+                                                run_tutorial_test
+                                                "01-vector-add"
+                                                run_tutorial_test
+                                                "02-fused-softmax"
+                                                run_tutorial_test
+                                                "03-matrix-multiplication"
+                                                run_tutorial_test
+                                                "04-low-memory-dropout"
+                                                run_tutorial_test
+                                                "05-layer-norm"
+                                                run_tutorial_test
+                                                "06-fused-attention"
+                                                run_tutorial_test
+                                                "07-extern-functions"
+                                                run_tutorial_test
+                                                "08-grouped-gemm"
+                                                run_tutorial_test
+                                                "10-experimental-block-pointer"
+                                                run_tutorial_test
+                                                "10i-experimental-block-pointer"
+                                        }
+
+                                        run_microbench_tests() { echo
+                                                "****************************************************"
+                                                echo "*****   Running Triton
+                                                Micro Benchmark tests   *****"
+                                                echo
+                                                "****************************************************"
+                                                USE_IPEX=0 python
+                                                $TRITON_PROJ/benchmarks/micro_benchmarks/run_benchmarks.py
+                                        }
+
+                                        run_benchmark_softmax() { echo
+                                                "****************************************************"
+                                                echo "*****             Running
+                                                Softmax              *****"
+                                                echo
+                                                "****************************************************"
+                                                cd $TRITON_PROJ/benchmarks
+                                                python setup.py install python
+                                                $TRITON_PROJ/benchmarks/triton_kernels_benchmark/fused_softmax.py
+                                        }
+
+                                        run_benchmark_gemm() { echo
+                                                "****************************************************"
+                                                echo "*****
+                                                Running GEMM
+                                                *****" echo
+                                                "****************************************************"
+                                                cd $TRITON_PROJ/benchmarks
+                                                python setup.py install
+
+                                                echo "Default path:"
+                                                TRITON_INTEL_ADVANCED_PATH=0 \
+                                                        TRITON_INTEL_ENABLE_ADDRESS_PAYLOAD_OPT=1
+                                                                                                        \
+                                                                                                                IGC_VISAOptions="
+                                                                                                                                                                                                                        -enableBCR
+                                                                                                                                                                                                                        -nolocalra"
+                                                                                                                                                                                                                        \
+                                                                                                                                                                                                                                IGC_DisableLoopUnroll=1
+                                                                                                                                                                                                                                                                                                                                                                                                                                                        \
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                python
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        $TRITON_PROJ/benchmarks/triton_kernels_benchmark/gemm_benchmark.py
+
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        echo "Advanced path:" TRITON_INTEL_ADVANCED_PATH=1 \ TRITON_INTEL_ENABLE_ADDRESS_PAYLOAD_OPT=1 \ IGC_VISAOptions=" -enableBCR -nolocalra" \ IGC_DisableLoopUnroll=1 \ python $TRITON_PROJ/benchmarks/triton_kernels_benchmark/gemm_benchmark.py }
+
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        run_benchmark_attention() { echo "****************************************************" echo "*****            Running ATTENTION             *****" echo "****************************************************" cd $TRITON_PROJ/benchmarks python setup.py install
+
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                echo "Default path:" TRITON_INTEL_ADVANCED_PATH=0 \ TRITON_INTEL_ENABLE_ADDRESS_PAYLOAD_OPT=1 \ IGC_VISAOptions=" -enableBCR" \ python $TRITON_PROJ/benchmarks/triton_kernels_benchmark/flash_attention_fwd_benchmark.py
+
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                echo "Advanced path:" TRITON_INTEL_ADVANCED_PATH=1 \ TRITON_INTEL_ENABLE_ADDRESS_PAYLOAD_OPT=1 \ TRITON_INTEL_ENABLE_INSTR_SCHED=1 \ IGC_VISAOptions=" -enableBCR" \ python $TRITON_PROJ/benchmarks/triton_kernels_benchmark/flash_attention_fwd_benchmark.py }
+
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                run_instrumentation_tests() {
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        # FIXME: the "instrumentation" test suite currently contains only one test, when all tests are skipped pytest reports an error. If the only test is the skip list, then we shouldn't run pytest at all. This must be changed when there is more than one instrumentation test.
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        if [[ $TEST_UNSKIP = false && -s $TRITON_TEST_SKIPLIST_DIR/instrumentation.txt ]]; then return fi
+
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        SHARED_LIB_DIR=$(ls -1d $TRITON_PROJ/python/build/*lib*/triton/_C) || SHARED_LIB_DIR=$(ls -1d $TRITON_PROJ/python/triton/_C) || err "Could not find $TRITON_PROJ/python/build/*lib*/triton/_C, build Triton first"
+
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        cd $TRITON_PROJ/python/test/unit
+
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        TRITON_TEST_SUITE=instrumentation \ TRITON_ALWAYS_COMPILE=1 TRITON_DISABLE_LINE_INFO=0 LLVM_PASS_PLUGIN_PATH=${SHARED_LIB_DIR}/libGPUHello.so \ pytest -vvv --device xpu instrumentation/test_gpuhello.py }
 
 run_inductor_tests() {
-  test -d pytorch || {
+  test -d pytorch || (
     git clone https://github.com/pytorch/pytorch
     rev=$(cat .github/pins/pytorch-upstream.txt)
     cd pytorch
     git checkout $rev
-  }
+  )
 
   pip install pyyaml pandas scipy numpy psutil pyre_extensions torchrec
 

From 5719ed9b1d44f9ff7c6eaea450d8fa881b695148 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Thu, 17 Oct 2024 22:13:08 +0200
Subject: [PATCH 089/105] Revert vim autoformatting

---
 scripts/test-triton.sh | 558 ++++++++++++++++++++++-------------------
 1 file changed, 297 insertions(+), 261 deletions(-)

diff --git a/scripts/test-triton.sh b/scripts/test-triton.sh
index ec9441a3ac..3dc456fa8b 100755
--- a/scripts/test-triton.sh
+++ b/scripts/test-triton.sh
@@ -2,279 +2,315 @@
 
 set -euo pipefail
 
-err() { echo $@ exit 1 }
+err() {
+    echo $@
+    exit 1
+}
 
 export PIP_DISABLE_PIP_VERSION_CHECK=1
 
 # Select which tests to run.
-TEST_UNIT=false TEST_CORE=false TEST_INTERPRETER=false TEST_TUTORIAL=false
-TEST_MICRO_BENCHMARKS=false TEST_BENCHMARK_SOFTMAX=false
-TEST_BENCHMARK_GEMM=false TEST_BENCHMARK_ATTENTION=false
-TEST_INSTRUMENTATION=false TEST_INDUCTOR=false VENV=false
-TRITON_TEST_REPORTS=false TRITON_TEST_WARNING_REPORTS=false
-TRITON_TEST_IGNORE_ERRORS=false SKIP_PIP=false SKIP_PYTORCH=false
-TEST_UNSKIP=false while [ -v 1 ]; do case "$1" in --unskip) TEST_UNSKIP=true
-shift ;; --unit) TEST_UNIT=true shift ;; --core) TEST_CORE=true shift ;;
---interpreter) TEST_INTERPRETER=true shift ;; --tutorial) TEST_TUTORIAL=true
-        shift ;; --microbench) TEST_MICRO_BENCHMARKS=true shift ;; --softmax)
-        TEST_BENCHMARK_SOFTMAX=true shift ;; --gemm) TEST_BENCHMARK_GEMM=true
-        shift ;; --attention) TEST_BENCHMARK_ATTENTION=true shift ;;
-        --instrumentation) TEST_INSTRUMENTATION=true shift ;; --inductor)
-                TEST_INDUCTOR=true shift ;; --venv) VENV=true shift ;;
-                --skip-pip-install) SKIP_PIP=true shift ;;
-                --skip-pytorch-install) SKIP_PYTORCH=true shift ;; --reports)
-                        TRITON_TEST_REPORTS=true shift ;; --reports-dir)
-                        TRITON_TEST_REPORTS=true TRITON_TEST_REPORTS_DIR="$2"
-                        shift 2 ;; --warning-reports)
-                        TRITON_TEST_WARNING_REPORTS=true shift ;;
-                --ignore-errors) TRITON_TEST_IGNORE_ERRORS=true shift ;;
-                --skip-list) TRITON_TEST_SKIPLIST_DIR="$2" shift 2 ;; --help)
-                        err "Example usage: ./test-triton.sh [--core |
-                                --tutorial | --unit | --microbench | --softmax
-                                                        | --gemm | --attention
-                                                        | --venv |
-                                                                --skip-pip-install
-                                                                                                                        |
-                                                                                                                                --skip-pytorch-install
-                                                                                                                                                                                                                                                        |
-                                                                                                                                                                                                                                                                --reports
-                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
-                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                --reports-dir
-                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        DIR | --warning-reports | --ignore-errors | --skip-list SKIPLIST" ;; *) err "Unknown argument: $1." ;; esac done
+TEST_UNIT=false
+TEST_CORE=false
+TEST_INTERPRETER=false
+TEST_TUTORIAL=false
+TEST_MICRO_BENCHMARKS=false
+TEST_BENCHMARK_SOFTMAX=false
+TEST_BENCHMARK_GEMM=false
+TEST_BENCHMARK_ATTENTION=false
+TEST_INSTRUMENTATION=false
+TEST_INDUCTOR=false
+VENV=false
+TRITON_TEST_REPORTS=false
+TRITON_TEST_WARNING_REPORTS=false
+TRITON_TEST_IGNORE_ERRORS=false
+SKIP_PIP=false
+SKIP_PYTORCH=false
+TEST_UNSKIP=false
+while [ -v 1 ]; do
+  case "$1" in
+    --unskip)
+      TEST_UNSKIP=true
+      shift
+      ;;
+    --unit)
+      TEST_UNIT=true
+      shift
+      ;;
+    --core)
+      TEST_CORE=true
+      shift
+      ;;
+    --interpreter)
+      TEST_INTERPRETER=true
+      shift
+      ;;
+    --tutorial)
+      TEST_TUTORIAL=true
+      shift
+      ;;
+    --microbench)
+      TEST_MICRO_BENCHMARKS=true
+      shift
+      ;;
+    --softmax)
+      TEST_BENCHMARK_SOFTMAX=true
+      shift
+      ;;
+    --gemm)
+      TEST_BENCHMARK_GEMM=true
+      shift
+      ;;
+    --attention)
+      TEST_BENCHMARK_ATTENTION=true
+      shift
+      ;;
+    --instrumentation)
+      TEST_INSTRUMENTATION=true
+      shift
+      ;;
+    --inductor)
+      TEST_INDUCTOR=true
+      shift
+      ;;
+    --venv)
+      VENV=true
+      shift
+      ;;
+    --skip-pip-install)
+      SKIP_PIP=true
+      shift
+      ;;
+    --skip-pytorch-install)
+      SKIP_PYTORCH=true
+      shift
+      ;;
+    --reports)
+      TRITON_TEST_REPORTS=true
+      shift
+      ;;
+    --reports-dir)
+      TRITON_TEST_REPORTS=true
+      TRITON_TEST_REPORTS_DIR="$2"
+      shift 2
+      ;;
+    --warning-reports)
+      TRITON_TEST_WARNING_REPORTS=true
+      shift
+      ;;
+    --ignore-errors)
+      TRITON_TEST_IGNORE_ERRORS=true
+      shift
+      ;;
+    --skip-list)
+      TRITON_TEST_SKIPLIST_DIR="$2"
+      shift 2
+      ;;
+    --help)
+      err "Example usage: ./test-triton.sh [--core | --tutorial | --unit | --microbench | --softmax | --gemm | --attention | --venv | --skip-pip-install | --skip-pytorch-install | --reports | --reports-dir DIR | --warning-reports | --ignore-errors | --skip-list SKIPLIST"
+      ;;
+    *)
+      err "Unknown argument: $1."
+      ;;
+  esac
+done
 
 # Only run interpreter test when $TEST_INTERPRETER is true
-if [ "$TEST_UNIT" = false ] && [ "$TEST_CORE" = false ] && [
-        "$TEST_INTERPRETER" = false ] && [ "$TEST_TUTORIAL" = false ] && [
-        "$TEST_MICRO_BENCHMARKS" = false ] && [ "$TEST_BENCHMARK_SOFTMAX" =
-        false ] && [ "$TEST_BENCHMARK_GEMM" = false ] && [
-        "$TEST_BENCHMARK_ATTENTION" = false ] && [ "$TEST_INSTRUMENTATION" =
-        false ] && [ "$TEST_INDUCTOR" = false ]; then TEST_UNIT=true
-        TEST_CORE=true TEST_TUTORIAL=true TEST_MICRO_BENCHMARKS=true fi
-
-        if [ "$VENV" = true ]; then source .venv/bin/activate fi
-
-
-        TRITON_PROJ="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &>
-        /dev/null && cd .. && pwd )" SCRIPTS_DIR="$TRITON_PROJ/scripts" source
-        "$SCRIPTS_DIR/pytest-utils.sh"
-
-        if [ "$TRITON_TEST_REPORTS" == true ]; then capture_runtime_env fi
-
-        install_deps() { if [ "$SKIP_PIP" = true ]; then echo "**** Skipping
-                installation of pip dependencies ****" else echo "****
-                Installing pip dependencies ****" python -m pip install -r
-                "$SCRIPTS_DIR/requirements-test.txt"
-
-                if [ "$TRITON_TEST_WARNING_REPORTS" == true ]; then python -m
-                        pip install
-                        git+https://github.com/kwasd/pytest-capturewarnings-ng@v1.2.0
-                        fi fi
-
-                        if [ "$SKIP_PYTORCH" = true ]; then echo "**** Skipping
-                                installation of pytorch ****" else echo "****
-                                Installing pytorch ****" if [
-                                "$TEST_BENCHMARK_SOFTMAX" = true ] || [
-                                "$TEST_BENCHMARK_GEMM" = true ] || [
-                                "$TEST_BENCHMARK_ATTENTION" = true ]; then
-                                $SCRIPTS_DIR/compile-pytorch-ipex.sh $([ $VENV
-                                = true ] && echo "--venv") else
-                                $SCRIPTS_DIR/install-pytorch.sh $([ $VENV =
-                                true ] && echo "--venv") fi fi }
-
-                                run_unit_tests() { echo
-                                        "***************************************************"
-                                        echo "******      Running Triton CXX
-                                        unittests     ******" echo
-                                        "***************************************************"
-                                        cd $TRITON_PROJ/python/build/cmake* ||
-                                                err "****** ERROR: Build Triton
-                                                                                        first
-                                                                                        ******"
-                                                                                        ctest
-                                                                                        .
-
-                                                                                        echo
-                                                                                        "***************************************************"
-                                                                                        echo
-                                                                                        "******
-                                                                                        Running
-                                                                                        Triton
-                                                                                        LIT
-                                                                                        tests
-                                                                                        ******"
-                                                                                        echo
-                                                                                        "***************************************************"
-                                                                                        cd
-                                                                                        $TRITON_PROJ/python/build/cmake*/test
-                                                                                        lit
-                                                                                        -v
-                                                                                        .
-                                                                                }
-
-                                                                                run_core_tests()
-                                                                                {
-                                                                                        echo
-                                                                                        "***************************************************"
-                                                                                        echo
-                                                                                        "******
-                                                                                        Running
-                                                                                        Triton
-                                                                                        Core
-                                                                                        tests
-                                                                                        ******"
-                                                                                        echo
-                                                                                        "***************************************************"
-                                                                                        cd
-                                                                                        $TRITON_PROJ/python/test/unit
-                                                                                        ensure_spirv_dis
-
-                                                                                        TRITON_DISABLE_LINE_INFO=1
-                                                                                        TRITON_TEST_SUITE=language
-                                                                                        \
-                                                                                                pytest
-                                                                                                                                                                                        -vvv
-                                                                                                                                                                                        -n
-                                                                                                                                                                                        ${PYTEST_MAX_PROCESSES:-8}
-                                                                                                                                                                                        --device
-                                                                                                                                                                                        xpu
-                                                                                                                                                                                        language/
-                                                                                                                                                                                        --ignore=language/test_line_info.py
-                                                                                                                                                                                        --ignore=language/test_subprocess.py
-
-                                                                                                                                                                                        TRITON_DISABLE_LINE_INFO=1
-                                                                                                                                                                                        TRITON_TEST_SUITE=subprocess
-                                                                                                                                                                                        \
-                                                                                                                                                                                                pytest
-                                                                                                                                                                                                                                                                                                                                                                                        -vvv
-                                                                                                                                                                                                                                                                                                                                                                                        -n
-                                                                                                                                                                                                                                                                                                                                                                                        ${PYTEST_MAX_PROCESSES:-8}
-                                                                                                                                                                                                                                                                                                                                                                                        --device
-                                                                                                                                                                                                                                                                                                                                                                                        xpu
-                                                                                                                                                                                                                                                                                                                                                                                        language/test_subprocess.py
+if [ "$TEST_UNIT" = false ] && [ "$TEST_CORE" = false ] && [ "$TEST_INTERPRETER" = false ] && [ "$TEST_TUTORIAL" = false ] && [ "$TEST_MICRO_BENCHMARKS" = false ] && [ "$TEST_BENCHMARK_SOFTMAX" = false ] && [ "$TEST_BENCHMARK_GEMM" = false ] && [ "$TEST_BENCHMARK_ATTENTION" = false ] && [ "$TEST_INSTRUMENTATION" = false ] && [ "$TEST_INDUCTOR" = false ]; then
+  TEST_UNIT=true
+  TEST_CORE=true
+  TEST_TUTORIAL=true
+  TEST_MICRO_BENCHMARKS=true
+fi
+
+if [ "$VENV" = true ]; then
+  source .venv/bin/activate
+fi
+
+
+TRITON_PROJ="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && cd .. && pwd )"
+SCRIPTS_DIR="$TRITON_PROJ/scripts"
+source "$SCRIPTS_DIR/pytest-utils.sh"
+
+if [ "$TRITON_TEST_REPORTS" == true ]; then
+    capture_runtime_env
+fi
+
+install_deps() {
+  if [ "$SKIP_PIP" = true ]; then
+    echo "**** Skipping installation of pip dependencies ****"
+  else
+    echo "**** Installing pip dependencies ****"
+    python -m pip install -r "$SCRIPTS_DIR/requirements-test.txt"
+
+    if [ "$TRITON_TEST_WARNING_REPORTS" == true ]; then
+      python -m pip install git+https://github.com/kwasd/pytest-capturewarnings-ng@v1.2.0
+    fi
+  fi
+
+  if [ "$SKIP_PYTORCH" = true ]; then
+    echo "**** Skipping installation of pytorch ****"
+  else
+    echo "**** Installing pytorch ****"
+    if [ "$TEST_BENCHMARK_SOFTMAX" = true ] || [ "$TEST_BENCHMARK_GEMM" = true ] || [ "$TEST_BENCHMARK_ATTENTION" = true ]; then
+      $SCRIPTS_DIR/compile-pytorch-ipex.sh $([ $VENV = true ] && echo "--venv")
+    else
+      $SCRIPTS_DIR/install-pytorch.sh $([ $VENV = true ] && echo "--venv")
+    fi
+  fi
+}
+
+run_unit_tests() {
+  echo "***************************************************"
+  echo "******      Running Triton CXX unittests     ******"
+  echo "***************************************************"
+  cd $TRITON_PROJ/python/build/cmake* || err "****** ERROR: Build Triton first ******"
+  ctest .
+
+  echo "***************************************************"
+  echo "******       Running Triton LIT tests        ******"
+  echo "***************************************************"
+  cd $TRITON_PROJ/python/build/cmake*/test
+  lit -v .
+}
+
+run_core_tests() {
+  echo "***************************************************"
+  echo "******      Running Triton Core tests        ******"
+  echo "***************************************************"
+  cd $TRITON_PROJ/python/test/unit
+  ensure_spirv_dis
+
+  TRITON_DISABLE_LINE_INFO=1 TRITON_TEST_SUITE=language \
+    pytest -vvv -n ${PYTEST_MAX_PROCESSES:-8} --device xpu language/ --ignore=language/test_line_info.py --ignore=language/test_subprocess.py
+
+  TRITON_DISABLE_LINE_INFO=1 TRITON_TEST_SUITE=subprocess \
+    pytest -vvv -n ${PYTEST_MAX_PROCESSES:-8} --device xpu language/test_subprocess.py
 
   # run runtime tests serially to avoid race condition with cache handling.
-  TRITON_DISABLE_LINE_INFO=1 TRITON_TEST_SUITE=runtime \ pytest --verbose
-  --device xpu runtime/
+  TRITON_DISABLE_LINE_INFO=1 TRITON_TEST_SUITE=runtime \
+    pytest --verbose --device xpu runtime/
 
   # run test_line_info.py separately with TRITON_DISABLE_LINE_INFO=0
-  TRITON_DISABLE_LINE_INFO=0 TRITON_TEST_SUITE=line_info \ pytest -k "not
-  test_line_info_interpreter" --verbose --device xpu language/test_line_info.py
+  TRITON_DISABLE_LINE_INFO=0 TRITON_TEST_SUITE=line_info \
+    pytest -k "not test_line_info_interpreter" --verbose --device xpu language/test_line_info.py
+}
+
+run_regression_tests() {
+  echo "***************************************************"
+  echo "******   Running Triton Regression tests     ******"
+  echo "***************************************************"
+  cd $TRITON_PROJ/python/test/regression
+
+  TRITON_DISABLE_LINE_INFO=1 TRITON_TEST_SUITE=regression \
+    pytest -vvv -s --device xpu . --ignore=test_performance.py
+}
+
+run_interpreter_tests() {
+  echo "***************************************************"
+  echo "******   Running Triton Interpreter tests    ******"
+  echo "***************************************************"
+  cd $TRITON_PROJ/python/test/unit
+
+  TRITON_INTERPRET=1 TRITON_TEST_SUITE=interpreter \
+    pytest -vvv -n ${PYTEST_MAX_PROCESSES:-16} -m interpreter language/test_core.py language/test_standard.py \
+    language/test_random.py language/test_line_info.py --device cpu
 }
 
-run_regression_tests() { echo
-        "***************************************************" echo "******
-        Running Triton Regression tests     ******" echo
-        "***************************************************" cd
-        $TRITON_PROJ/python/test/regression
-
-        TRITON_DISABLE_LINE_INFO=1 TRITON_TEST_SUITE=regression \ pytest -vvv
-        -s --device xpu . --ignore=test_performance.py }
-
-        run_interpreter_tests() { echo
-                "***************************************************" echo
-                "******   Running Triton Interpreter tests    ******" echo
-                "***************************************************" cd
-                $TRITON_PROJ/python/test/unit
-
-                TRITON_INTERPRET=1 TRITON_TEST_SUITE=interpreter \ pytest -vvv
-                -n ${PYTEST_MAX_PROCESSES:-16} -m interpreter
-                language/test_core.py language/test_standard.py \
-                        language/test_random.py language/test_line_info.py
-                                        --device cpu }
-
-                                        run_tutorial_tests() { echo
-                                                "***************************************************"
-                                                echo "**** Running Triton
-                                                Tutorial tests
-                                                ******" echo
-                                                "***************************************************"
-                                                python -m pip install
-                                                matplotlib pandas tabulate -q
-                                                cd
-                                                $TRITON_PROJ/python/tutorials
-
-                                                run_tutorial_test
-                                                "01-vector-add"
-                                                run_tutorial_test
-                                                "02-fused-softmax"
-                                                run_tutorial_test
-                                                "03-matrix-multiplication"
-                                                run_tutorial_test
-                                                "04-low-memory-dropout"
-                                                run_tutorial_test
-                                                "05-layer-norm"
-                                                run_tutorial_test
-                                                "06-fused-attention"
-                                                run_tutorial_test
-                                                "07-extern-functions"
-                                                run_tutorial_test
-                                                "08-grouped-gemm"
-                                                run_tutorial_test
-                                                "10-experimental-block-pointer"
-                                                run_tutorial_test
-                                                "10i-experimental-block-pointer"
-                                        }
-
-                                        run_microbench_tests() { echo
-                                                "****************************************************"
-                                                echo "*****   Running Triton
-                                                Micro Benchmark tests   *****"
-                                                echo
-                                                "****************************************************"
-                                                USE_IPEX=0 python
-                                                $TRITON_PROJ/benchmarks/micro_benchmarks/run_benchmarks.py
-                                        }
-
-                                        run_benchmark_softmax() { echo
-                                                "****************************************************"
-                                                echo "*****             Running
-                                                Softmax              *****"
-                                                echo
-                                                "****************************************************"
-                                                cd $TRITON_PROJ/benchmarks
-                                                python setup.py install python
-                                                $TRITON_PROJ/benchmarks/triton_kernels_benchmark/fused_softmax.py
-                                        }
-
-                                        run_benchmark_gemm() { echo
-                                                "****************************************************"
-                                                echo "*****
-                                                Running GEMM
-                                                *****" echo
-                                                "****************************************************"
-                                                cd $TRITON_PROJ/benchmarks
-                                                python setup.py install
-
-                                                echo "Default path:"
-                                                TRITON_INTEL_ADVANCED_PATH=0 \
-                                                        TRITON_INTEL_ENABLE_ADDRESS_PAYLOAD_OPT=1
-                                                                                                        \
-                                                                                                                IGC_VISAOptions="
-                                                                                                                                                                                                                        -enableBCR
-                                                                                                                                                                                                                        -nolocalra"
-                                                                                                                                                                                                                        \
-                                                                                                                                                                                                                                IGC_DisableLoopUnroll=1
-                                                                                                                                                                                                                                                                                                                                                                                                                                                        \
-                                                                                                                                                                                                                                                                                                                                                                                                                                                                python
-                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        $TRITON_PROJ/benchmarks/triton_kernels_benchmark/gemm_benchmark.py
-
-                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        echo "Advanced path:" TRITON_INTEL_ADVANCED_PATH=1 \ TRITON_INTEL_ENABLE_ADDRESS_PAYLOAD_OPT=1 \ IGC_VISAOptions=" -enableBCR -nolocalra" \ IGC_DisableLoopUnroll=1 \ python $TRITON_PROJ/benchmarks/triton_kernels_benchmark/gemm_benchmark.py }
-
-                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        run_benchmark_attention() { echo "****************************************************" echo "*****            Running ATTENTION             *****" echo "****************************************************" cd $TRITON_PROJ/benchmarks python setup.py install
-
-                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                echo "Default path:" TRITON_INTEL_ADVANCED_PATH=0 \ TRITON_INTEL_ENABLE_ADDRESS_PAYLOAD_OPT=1 \ IGC_VISAOptions=" -enableBCR" \ python $TRITON_PROJ/benchmarks/triton_kernels_benchmark/flash_attention_fwd_benchmark.py
-
-                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                echo "Advanced path:" TRITON_INTEL_ADVANCED_PATH=1 \ TRITON_INTEL_ENABLE_ADDRESS_PAYLOAD_OPT=1 \ TRITON_INTEL_ENABLE_INSTR_SCHED=1 \ IGC_VISAOptions=" -enableBCR" \ python $TRITON_PROJ/benchmarks/triton_kernels_benchmark/flash_attention_fwd_benchmark.py }
-
-                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                run_instrumentation_tests() {
-                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        # FIXME: the "instrumentation" test suite currently contains only one test, when all tests are skipped pytest reports an error. If the only test is the skip list, then we shouldn't run pytest at all. This must be changed when there is more than one instrumentation test.
-                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        if [[ $TEST_UNSKIP = false && -s $TRITON_TEST_SKIPLIST_DIR/instrumentation.txt ]]; then return fi
-
-                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        SHARED_LIB_DIR=$(ls -1d $TRITON_PROJ/python/build/*lib*/triton/_C) || SHARED_LIB_DIR=$(ls -1d $TRITON_PROJ/python/triton/_C) || err "Could not find $TRITON_PROJ/python/build/*lib*/triton/_C, build Triton first"
-
-                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        cd $TRITON_PROJ/python/test/unit
-
-                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        TRITON_TEST_SUITE=instrumentation \ TRITON_ALWAYS_COMPILE=1 TRITON_DISABLE_LINE_INFO=0 LLVM_PASS_PLUGIN_PATH=${SHARED_LIB_DIR}/libGPUHello.so \ pytest -vvv --device xpu instrumentation/test_gpuhello.py }
+run_tutorial_tests() {
+  echo "***************************************************"
+  echo "**** Running Triton Tutorial tests           ******"
+  echo "***************************************************"
+  python -m pip install matplotlib pandas tabulate -q
+  cd $TRITON_PROJ/python/tutorials
+
+  run_tutorial_test "01-vector-add"
+  run_tutorial_test "02-fused-softmax"
+  run_tutorial_test "03-matrix-multiplication"
+  run_tutorial_test "04-low-memory-dropout"
+  run_tutorial_test "05-layer-norm"
+  run_tutorial_test "06-fused-attention"
+  run_tutorial_test "07-extern-functions"
+  run_tutorial_test "08-grouped-gemm"
+  run_tutorial_test "10-experimental-block-pointer"
+  run_tutorial_test "10i-experimental-block-pointer"
+}
+
+run_microbench_tests() {
+  echo "****************************************************"
+  echo "*****   Running Triton Micro Benchmark tests   *****"
+  echo "****************************************************"
+  USE_IPEX=0 python $TRITON_PROJ/benchmarks/micro_benchmarks/run_benchmarks.py
+}
+
+run_benchmark_softmax() {
+  echo "****************************************************"
+  echo "*****             Running Softmax              *****"
+  echo "****************************************************"
+  cd $TRITON_PROJ/benchmarks
+  python setup.py install
+  python $TRITON_PROJ/benchmarks/triton_kernels_benchmark/fused_softmax.py
+}
+
+run_benchmark_gemm() {
+  echo "****************************************************"
+  echo "*****              Running GEMM                *****"
+  echo "****************************************************"
+  cd $TRITON_PROJ/benchmarks
+  python setup.py install
+
+  echo "Default path:"
+  TRITON_INTEL_ADVANCED_PATH=0 \
+    TRITON_INTEL_ENABLE_ADDRESS_PAYLOAD_OPT=1 \
+    IGC_VISAOptions=" -enableBCR -nolocalra" \
+    IGC_DisableLoopUnroll=1 \
+    python $TRITON_PROJ/benchmarks/triton_kernels_benchmark/gemm_benchmark.py
+
+  echo "Advanced path:"
+  TRITON_INTEL_ADVANCED_PATH=1 \
+    TRITON_INTEL_ENABLE_ADDRESS_PAYLOAD_OPT=1 \
+    IGC_VISAOptions=" -enableBCR -nolocalra" \
+    IGC_DisableLoopUnroll=1 \
+    python $TRITON_PROJ/benchmarks/triton_kernels_benchmark/gemm_benchmark.py
+}
+
+run_benchmark_attention() {
+  echo "****************************************************"
+  echo "*****            Running ATTENTION             *****"
+  echo "****************************************************"
+  cd $TRITON_PROJ/benchmarks
+  python setup.py install
+
+  echo "Default path:"
+  TRITON_INTEL_ADVANCED_PATH=0 \
+    TRITON_INTEL_ENABLE_ADDRESS_PAYLOAD_OPT=1 \
+    IGC_VISAOptions=" -enableBCR" \
+    python $TRITON_PROJ/benchmarks/triton_kernels_benchmark/flash_attention_fwd_benchmark.py
+
+  echo "Advanced path:"
+  TRITON_INTEL_ADVANCED_PATH=1 \
+    TRITON_INTEL_ENABLE_ADDRESS_PAYLOAD_OPT=1 \
+    TRITON_INTEL_ENABLE_INSTR_SCHED=1 \
+    IGC_VISAOptions=" -enableBCR" \
+    python $TRITON_PROJ/benchmarks/triton_kernels_benchmark/flash_attention_fwd_benchmark.py
+}
+
+run_instrumentation_tests() {
+  # FIXME: the "instrumentation" test suite currently contains only one test, when all tests
+  # are skipped pytest reports an error. If the only test is the skip list, then we shouldn't
+  # run pytest at all. This must be changed when there is more than one instrumentation test.
+  if [[ $TEST_UNSKIP = false && -s $TRITON_TEST_SKIPLIST_DIR/instrumentation.txt ]]; then
+    return
+  fi
+
+  SHARED_LIB_DIR=$(ls -1d $TRITON_PROJ/python/build/*lib*/triton/_C) || SHARED_LIB_DIR=$(ls -1d $TRITON_PROJ/python/triton/_C) || err "Could not find $TRITON_PROJ/python/build/*lib*/triton/_C, build Triton first"
+
+  cd $TRITON_PROJ/python/test/unit
+
+  TRITON_TEST_SUITE=instrumentation \
+    TRITON_ALWAYS_COMPILE=1 TRITON_DISABLE_LINE_INFO=0 LLVM_PASS_PLUGIN_PATH=${SHARED_LIB_DIR}/libGPUHello.so \
+    pytest -vvv --device xpu instrumentation/test_gpuhello.py
+}
 
 run_inductor_tests() {
   test -d pytorch || (

From 779272dceb102c7b466dce3bb251c5b936547ec8 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 21 Oct 2024 13:52:14 +0200
Subject: [PATCH 090/105] Try adding sycl to includes

---
 scripts/install-conda.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/install-conda.sh b/scripts/install-conda.sh
index 953b20f5f6..d742f03db7 100755
--- a/scripts/install-conda.sh
+++ b/scripts/install-conda.sh
@@ -7,6 +7,7 @@ install_env() {
     find /opt/intel/oneapi/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
     ln -snf /usr/include/level_zero $HOME/miniforge3/envs/triton/bin/../x86_64-conda-linux-gnu/sysroot/usr/include/level_zero
     find /usr -name libze_\* -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
+    ln -snf /opt/intel/oneapi/compiler/2024.1/include/sycl $HOME/miniforge3/envs/triton/include/sycl
 }
 
 print_env_info() {

From 7f06bd69dfdbd5fafb9c55c0749f9b33a62ff0ea Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 21 Oct 2024 15:00:33 +0200
Subject: [PATCH 091/105] Try adding sycl to includes, r1

---
 scripts/install-conda.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install-conda.sh b/scripts/install-conda.sh
index d742f03db7..3dca68a9bb 100755
--- a/scripts/install-conda.sh
+++ b/scripts/install-conda.sh
@@ -7,7 +7,7 @@ install_env() {
     find /opt/intel/oneapi/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
     ln -snf /usr/include/level_zero $HOME/miniforge3/envs/triton/bin/../x86_64-conda-linux-gnu/sysroot/usr/include/level_zero
     find /usr -name libze_\* -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
-    ln -snf /opt/intel/oneapi/compiler/2024.1/include/sycl $HOME/miniforge3/envs/triton/include/sycl
+    ln -snf /opt/intel/oneapi/compiler/2024.1/include/sycl $HOME/miniforge3/envs/triton/lib/python3.9/site-packages/triton/backends/intel/include
 }
 
 print_env_info() {

From 80663bcc06178a4bc7769d6522e1486f76e403e3 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 21 Oct 2024 15:06:45 +0200
Subject: [PATCH 092/105] Try adding sycl to includes, r2

---
 scripts/install-conda.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scripts/install-conda.sh b/scripts/install-conda.sh
index 3dca68a9bb..3f8778022e 100755
--- a/scripts/install-conda.sh
+++ b/scripts/install-conda.sh
@@ -1,13 +1,16 @@
 #!/bin/bash
 
 install_env() {
+    set -vx
     export PATH="$HOME/miniforge3/bin:$PATH"
     conda create -n triton --override-channels -c conda-forge python=$python_version.*
     conda env update -f scripts/triton.yml
     find /opt/intel/oneapi/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
     ln -snf /usr/include/level_zero $HOME/miniforge3/envs/triton/bin/../x86_64-conda-linux-gnu/sysroot/usr/include/level_zero
     find /usr -name libze_\* -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
-    ln -snf /opt/intel/oneapi/compiler/2024.1/include/sycl $HOME/miniforge3/envs/triton/lib/python3.9/site-packages/triton/backends/intel/include
+    ln -snf /opt/intel/oneapi/compiler/2024.1/include/sycl $HOME/miniforge3/envs/triton/lib/python$python_version/site-packages/triton/backends/intel/include
+    ls -l /home/runner/miniforge3/envs/triton/lib/python$python_version/site-packages/triton/backends/intel/include/
+    ls -l /home/runner/miniforge3/envs/triton/lib/python$python_version/site-packages/triton/backends/intel/include
 }
 
 print_env_info() {

From 3fd7c7454cf7f613b3cd437408a9377f15359127 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 21 Oct 2024 15:12:18 +0200
Subject: [PATCH 093/105] Try adding sycl to includes, r3

---
 scripts/install-conda.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/install-conda.sh b/scripts/install-conda.sh
index 3f8778022e..965ad29606 100755
--- a/scripts/install-conda.sh
+++ b/scripts/install-conda.sh
@@ -8,6 +8,7 @@ install_env() {
     find /opt/intel/oneapi/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
     ln -snf /usr/include/level_zero $HOME/miniforge3/envs/triton/bin/../x86_64-conda-linux-gnu/sysroot/usr/include/level_zero
     find /usr -name libze_\* -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
+    mkdir -p $HOME/miniforge3/envs/triton/lib/python$python_version/site-packages/triton/backends/intel/include
     ln -snf /opt/intel/oneapi/compiler/2024.1/include/sycl $HOME/miniforge3/envs/triton/lib/python$python_version/site-packages/triton/backends/intel/include
     ls -l /home/runner/miniforge3/envs/triton/lib/python$python_version/site-packages/triton/backends/intel/include/
     ls -l /home/runner/miniforge3/envs/triton/lib/python$python_version/site-packages/triton/backends/intel/include

From c3a96ee7c4e59ebfe94fa2d4f896d93e11f7f3d6 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 21 Oct 2024 15:19:56 +0200
Subject: [PATCH 094/105] Try adding sycl to includes, r4

---
 scripts/install-conda.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/install-conda.sh b/scripts/install-conda.sh
index 965ad29606..efc3a4e416 100755
--- a/scripts/install-conda.sh
+++ b/scripts/install-conda.sh
@@ -10,8 +10,7 @@ install_env() {
     find /usr -name libze_\* -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
     mkdir -p $HOME/miniforge3/envs/triton/lib/python$python_version/site-packages/triton/backends/intel/include
     ln -snf /opt/intel/oneapi/compiler/2024.1/include/sycl $HOME/miniforge3/envs/triton/lib/python$python_version/site-packages/triton/backends/intel/include
-    ls -l /home/runner/miniforge3/envs/triton/lib/python$python_version/site-packages/triton/backends/intel/include/
-    ls -l /home/runner/miniforge3/envs/triton/lib/python$python_version/site-packages/triton/backends/intel/include
+    ln -snf /opt/intel/oneapi/compiler/2024.1/include/sycl/CL $HOME/miniforge3/envs/triton/lib/python$python_version/site-packages/triton/backends/intel/include
 }
 
 print_env_info() {

From 24012e0d6423a559d1049d98a5135b21c29574ee Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 21 Oct 2024 15:24:03 +0200
Subject: [PATCH 095/105] Try adding sycl to includes, r5

---
 .github/workflows/build-test-reusable.yml | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 45bbcec9a5..40cc68cf5f 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -128,18 +128,13 @@ jobs:
           ref: ${{ inputs.pytorch_ref }}
           mode: ${{ inputs.pytorch_mode }}
 
-      - name: Setup PyTorch
-        if: inputs.env_manager != 'base'
-        run: |
-          pip install torch --index-url https://download.pytorch.org/whl/nightly/xpu
-
       - name: Setup Triton
         if: inputs.env_manager == 'base'
         uses: ./.github/actions/setup-triton
         with:
           build_llvm: ${{ inputs.build_llvm }}
 
-      - name: Setup Triton and Transformers
+      - name: Setup Triton, Torch and Transformers
         if: inputs.env_manager != 'base'
         uses: ./.github/actions/install-wheels
         with:

From af275afaeeb9161ed648de32a92b8d13b293ba85 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 21 Oct 2024 15:51:40 +0200
Subject: [PATCH 096/105] Try adding sycl to includes, r6

---
 .github/workflows/conda-build-test.yml | 6 +-----
 scripts/test-triton.sh                 | 2 +-
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml
index 95e2e9b9be..d9e5b58dd9 100644
--- a/.github/workflows/conda-build-test.yml
+++ b/.github/workflows/conda-build-test.yml
@@ -64,11 +64,7 @@ jobs:
       - name: Matrix
         id: matrix
         run: |
-          if [[ -n "${{ inputs.runner_label }}" ]]; then
-            matrix='{"python": ["3.9"]}'
-          else
-            matrix='{"python": ["3.9"], "driver": ["rolling", "lts"]}'
-          fi
+          matrix='{"python": ["3.9"]}'
           echo "matrix=$matrix" | tee -a $GITHUB_OUTPUT
 
   integration-tests:
diff --git a/scripts/test-triton.sh b/scripts/test-triton.sh
index fe03da62db..40f417db72 100755
--- a/scripts/test-triton.sh
+++ b/scripts/test-triton.sh
@@ -322,7 +322,7 @@ run_inductor_tests() {
 
   pip install pyyaml pandas scipy numpy psutil pyre_extensions torchrec
 
-  ZE_AFFINITY_MASK=0 python pytorch/benchmarks/dynamo/huggingface.py --accuracy --float32 -dxpu -n10 --no-skip --dashboard --inference --freezing --total-partitions 1 --partition-id 0 --only AlbertForMaskedLM --backend=inductor --timeout=4800 --output=inductor_log.csv
+  ZE_AFFINITY_MASK=0 python pytorch/benchmarks/dynamo/huggingface.py --accuracy --float32 -dxpu -n10 --no-skip --dashboard --inference --freezing --total-partitions 1 --partition-id 0 --only AlbertForMaskedLM --backend=inductor --timeout=4800 --output=$(pwd -P)/inductor_log.csv
 
   cat inductor_log.csv
   grep AlbertForMaskedLM inductor_log.csv | grep -q ,pass,

From 87bb3b0c67fba221be7d76501709ae0ac943b0ad Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 21 Oct 2024 16:07:22 +0200
Subject: [PATCH 097/105] Try adding sycl to includes, r7

---
 .github/workflows/conda-build-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-build-test.yml
index d9e5b58dd9..1b73630593 100644
--- a/.github/workflows/conda-build-test.yml
+++ b/.github/workflows/conda-build-test.yml
@@ -64,7 +64,7 @@ jobs:
       - name: Matrix
         id: matrix
         run: |
-          matrix='{"python": ["3.9"]}'
+          matrix='{"python": ["3.9"], "driver": ["rolling"]}'
           echo "matrix=$matrix" | tee -a $GITHUB_OUTPUT
 
   integration-tests:

From 0ac2bf8446ebde7a8b2b5cbef548dd8c14ef44dd Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 21 Oct 2024 16:20:49 +0200
Subject: [PATCH 098/105] Remove unit tests

---
 .github/workflows/build-test-reusable.yml | 11 ++++++-----
 scripts/install-conda.sh                  |  1 -
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 40cc68cf5f..3af58626b4 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -163,12 +163,8 @@ jobs:
             echo TRITON_TEST_CMD="bash -v -x scripts/test-triton.sh --warning-reports --skip-pytorch-install --reports-dir $GITHUB_WORKSPACE/reports ${{ inputs.ignore_errors && '--ignore-errors' || '' }} $skiplist"
           } | tee -a $GITHUB_ENV
 
-      - name: Run inductor test
-        run: |
-          #FIXME move back test when ready
-          ${{ env.TRITON_TEST_CMD }} --inductor #--skip-pip-install
-
       - name: Run unit tests
+        if: inputs.env_manager == 'base'
         run: |
           ${{ env.TRITON_TEST_CMD }} --unit
 
@@ -185,6 +181,7 @@ jobs:
           ${{ env.TRITON_TEST_CMD }} --tutorial --skip-pip-install
 
       - name: Run instrumentation tests
+        if: inputs.env_manager == 'base'
         run: |
           ${{ env.TRITON_TEST_CMD }} --instrumentation --skip-pip-install
 
@@ -204,6 +201,10 @@ jobs:
           ref: ${{ env.TRANSFORMERS_VERSION }}
           try-tag-prefix: v
 
+      - name: Run inductor test
+        run: |
+          ${{ env.TRITON_TEST_CMD }} --inductor --skip-pip-install
+
       - name: Save pip cache
         if: (inputs.env_manager == 'base') && (steps.pip-cache.outputs.status == 'miss')
         uses: ./.github/actions/save
diff --git a/scripts/install-conda.sh b/scripts/install-conda.sh
index efc3a4e416..a00d806c06 100755
--- a/scripts/install-conda.sh
+++ b/scripts/install-conda.sh
@@ -1,7 +1,6 @@
 #!/bin/bash
 
 install_env() {
-    set -vx
     export PATH="$HOME/miniforge3/bin:$PATH"
     conda create -n triton --override-channels -c conda-forge python=$python_version.*
     conda env update -f scripts/triton.yml

From 499f412778846fa5ff292d2fed5a6ed367d67772 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 21 Oct 2024 16:26:15 +0200
Subject: [PATCH 099/105] Remove unit tests, r1

---
 .github/workflows/build-test-reusable.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 3af58626b4..bdb5dfd79f 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -170,7 +170,7 @@ jobs:
 
       - name: Run core tests
         run: |
-          ${{ env.TRITON_TEST_CMD }} --core --skip-pip-install
+          ${{ env.TRITON_TEST_CMD }} --core
 
       - name: Run interpreter tests
         run: |

From 0efbdb99931b2d4938ed0a6a8339413bba130c6e Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 21 Oct 2024 16:35:34 +0200
Subject: [PATCH 100/105] Remove debug

---
 .github/workflows/build-test-reusable.yml | 4 ----
 scripts/inductor_xpu_test.sh              | 2 +-
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index bdb5dfd79f..6d8a75d971 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -87,10 +87,6 @@ jobs:
           path: $HOME/miniforge3/envs/triton
           key: conda-${{ inputs.env_manager }}-py${{ matrix.python }}-${{ hashFiles('scripts/triton.yml', 'python/pyproject.toml', 'python/setup.py') }}-${{ env.CACHE_NUMBER }}
 
-      - name: Fail
-        if: inputs.env_manager == 'base'
-        run: exit 1
-
       - name: Install Manager Environment
         shell: bash --noprofile --norc -eo pipefail {0}
         run: |
diff --git a/scripts/inductor_xpu_test.sh b/scripts/inductor_xpu_test.sh
index 4f1652afc3..7373f6506c 100755
--- a/scripts/inductor_xpu_test.sh
+++ b/scripts/inductor_xpu_test.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 # This script work for xpu / cuda device inductor tests
-set -xv
+
 SUITE=${1:-huggingface}     # huggingface / torchbench / timm_models
 DT=${2:-float32}            # float32 / float16 / amp_bf16 / amp_fp16
 MODE=${3:-inference}        # inference / training

From 370c4f5482207c95471daefb879f4b52bade7ddc Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Thu, 24 Oct 2024 09:55:09 +0200
Subject: [PATCH 101/105] Moved to a separate workflow

---
 .github/workflows/build-test-reusable.yml     |  63 ++----
 .github/workflows/conda-test-reusable.yml     | 203 ++++++++++++++++++
 .../{conda-build-test.yml => conda-test.yml}  |   2 +-
 3 files changed, 216 insertions(+), 52 deletions(-)
 create mode 100644 .github/workflows/conda-test-reusable.yml
 rename .github/workflows/{conda-build-test.yml => conda-test.yml} (97%)

diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
index 6d8a75d971..aca94112b9 100644
--- a/.github/workflows/build-test-reusable.yml
+++ b/.github/workflows/build-test-reusable.yml
@@ -56,10 +56,6 @@ on:
         description: Runner label for version
         type: string
         default: runner-0.0.19
-      env_manager:
-        description: Environment manager
-        default: base
-        type: string
 
 permissions: read-all
 
@@ -73,71 +69,48 @@ jobs:
     runs-on: ${{ fromJson(inputs.runner_label && format('["{0}"]', inputs.runner_label) || format('["{0}", "{1}", "{2}"]', inputs.device, inputs.driver_version, inputs.runner_version)) }}
     defaults:
       run:
-        shell: bash -noprofile --norc -eo pipefail scripts/run-${{ inputs.env_manager }}.sh {0}
+        shell: bash -noprofile --norc -eo pipefail -c "source /opt/intel/oneapi/setvars.sh > /dev/null; source {0}"
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Load conda cache
-        id: conda-cache
-        uses: ./.github/actions/load
-        env:
-          CACHE_NUMBER: 6
-        with:
-          path: $HOME/miniforge3/envs/triton
-          key: conda-${{ inputs.env_manager }}-py${{ matrix.python }}-${{ hashFiles('scripts/triton.yml', 'python/pyproject.toml', 'python/setup.py') }}-${{ env.CACHE_NUMBER }}
-
-      - name: Install Manager Environment
-        shell: bash --noprofile --norc -eo pipefail {0}
-        run: |
-          scripts/install-${{ inputs.env_manager }}.sh --python-version ${{ inputs.python_version }}
-          echo $HOME/miniforge3/bin >>$GITHUB_PATH
-
       - name: Print inputs
         run: |
           cat <<EOF
           ${{ toJSON(inputs) }}
           EOF
 
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
       - name: Load pip cache
-        if: inputs.env_manager == 'base'
         id: pip-cache
         uses: ./.github/actions/load
         env:
           # Increase this value to reset cache
-          CACHE_NUMBER: 2
+          CACHE_NUMBER: 1
         with:
           path: $HOME/.cache/pip
           key: pip-${{ inputs.python_version }}-${{ hashFiles('python/pyproject.toml', 'python/setup.py') }}-${{ env.CACHE_NUMBER }}
 
       - name: Install Python ${{ inputs.python_version }}
-        if: inputs.env_manager == 'base'
         uses: actions/setup-python@v5
         with:
           python-version: ${{ inputs.python_version }}
 
       - name: Setup PyTorch
-        if: inputs.env_manager == 'base'
         uses: ./.github/actions/setup-pytorch
         with:
           repository: pytorch/pytorch
           ref: ${{ inputs.pytorch_ref }}
           mode: ${{ inputs.pytorch_mode }}
 
+      - name: Install pass_rate dependencies
+        run: |
+          pip install defusedxml
+
       - name: Setup Triton
-        if: inputs.env_manager == 'base'
         uses: ./.github/actions/setup-triton
         with:
           build_llvm: ${{ inputs.build_llvm }}
 
-      - name: Setup Triton, Torch and Transformers
-        if: inputs.env_manager != 'base'
-        uses: ./.github/actions/install-wheels
-        with:
-          gh_token: ${{ secrets.GITHUB_TOKEN }}
-          install_cmd: conda run --no-capture-output -n triton pip install
-          python_version: ${{ inputs.python_version }}
-
       - name: Create test-triton command line
         run: |
           if [[ -n "${{ inputs.skip_list }}" ]]; then
@@ -160,13 +133,12 @@ jobs:
           } | tee -a $GITHUB_ENV
 
       - name: Run unit tests
-        if: inputs.env_manager == 'base'
         run: |
           ${{ env.TRITON_TEST_CMD }} --unit
 
       - name: Run core tests
         run: |
-          ${{ env.TRITON_TEST_CMD }} --core
+          ${{ env.TRITON_TEST_CMD }} --core --skip-pip-install
 
       - name: Run interpreter tests
         run: |
@@ -177,19 +149,16 @@ jobs:
           ${{ env.TRITON_TEST_CMD }} --tutorial --skip-pip-install
 
       - name: Run instrumentation tests
-        if: inputs.env_manager == 'base'
         run: |
           ${{ env.TRITON_TEST_CMD }} --instrumentation --skip-pip-install
 
       - name: Get transformers version
-        if: inputs.env_manager == 'base'
         run: |
           cd pytorch
           TRANSFORMERS_VERSION="$(<.ci/docker/ci_commit_pins/huggingface.txt)"
           echo "TRANSFORMERS_VERSION=$TRANSFORMERS_VERSION" | tee -a $GITHUB_ENV
 
       - name: Install transformers
-        if: inputs.env_manager == 'base'
         uses: ./.github/actions/install-dependency
         with:
           package: transformers
@@ -197,27 +166,19 @@ jobs:
           ref: ${{ env.TRANSFORMERS_VERSION }}
           try-tag-prefix: v
 
-      - name: Run inductor test
+      - name: Run E2E test
         run: |
           ${{ env.TRITON_TEST_CMD }} --inductor --skip-pip-install
 
       - name: Save pip cache
-        if: (inputs.env_manager == 'base') && (steps.pip-cache.outputs.status == 'miss')
+        if: ${{ steps.pip-cache.outputs.status == 'miss' }}
         uses: ./.github/actions/save
         with:
           path: ${{ steps.pip-cache.outputs.path }}
           dest: ${{ steps.pip-cache.outputs.dest }}
 
-      - name: Save conda cache
-        if: (inputs.env_manager != 'base') && (steps.conda-cache.outputs.status == 'miss')
-        uses: ./.github/actions/save
-        with:
-          path: ${{ steps.conda-cache.outputs.path }}
-          dest: ${{ steps.conda-cache.outputs.dest }}
-
       - name: Pass rate
         run: |
-          pip install defusedxml
           source ./scripts/capture-hw-details.sh
           python3 scripts/pass_rate.py --reports reports ${{ env.SKIPLIST }}
           python3 scripts/pass_rate.py --reports reports --json ${{ env.SKIPLIST }} > pass_rate.json
diff --git a/.github/workflows/conda-test-reusable.yml b/.github/workflows/conda-test-reusable.yml
new file mode 100644
index 0000000000..11989e8587
--- /dev/null
+++ b/.github/workflows/conda-test-reusable.yml
@@ -0,0 +1,203 @@
+name: Build and test reusable workflow
+run-name: ${{ inputs.run_name }} - ${{ inputs.python_version }} - ${{ inputs.runner_label || 'default'}}
+
+on:
+  workflow_call:
+    inputs:
+      device:
+        description: Device
+        type: string
+        default: max1100
+      driver_version:
+        description: Driver version
+        type: string
+        default: rolling
+      runner_label:
+        description: Runner label, keep empty for default
+        type: string
+        default: ""
+      pytorch_ref:
+        description: PyTorch ref, keep empty for default
+        type: string
+        default: ""
+      pytorch_mode:
+        description: PyTorch mode, source or wheels
+        type: string
+        default: "source"
+      python_version:
+        description: Python version
+        type: string
+        required: true
+      upload_test_reports:
+        description: Upload test reports
+        type: boolean
+        default: false
+      ignore_errors:
+        description: Ignore test errors
+        type: boolean
+        default: false
+      skip_list:
+        description: Skip list
+        type: string
+        default: ""
+      run_name:
+        description: Custom run name
+        type: string
+        default: Build and test
+      build_llvm:
+        description: Build LLVM
+        type: boolean
+        default: false
+      enable_unskip:
+        description: Ignore pytest.skip
+        type: boolean
+        default: false
+      runner_version:
+        description: Runner label for version
+        type: string
+        default: runner-0.0.19
+      env_manager:
+        description: Environment manager
+        default: conda
+        type: string
+
+permissions: read-all
+
+env:
+  TRITON_DISABLE_LINE_INFO: 1
+  TEST_UNSKIP: ${{ inputs.enable_unskip }}
+
+jobs:
+  integration-tests:
+    name: Integration tests
+    runs-on: ${{ fromJson(inputs.runner_label && format('["{0}"]', inputs.runner_label) || format('["{0}", "{1}", "{2}"]', inputs.device, inputs.driver_version, inputs.runner_version)) }}
+    defaults:
+      run:
+        shell: bash -noprofile --norc -eo pipefail scripts/run-${{ inputs.env_manager }}.sh {0}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Load conda cache
+        id: conda-cache
+        uses: ./.github/actions/load
+        env:
+          CACHE_NUMBER: 6
+        with:
+          path: $HOME/miniforge3/envs/triton
+          key: conda-${{ inputs.env_manager }}-py${{ matrix.python }}-${{ hashFiles('scripts/triton.yml', 'python/pyproject.toml', 'python/setup.py') }}-${{ env.CACHE_NUMBER }}
+
+      - name: Install Manager Environment
+        shell: bash --noprofile --norc -eo pipefail {0}
+        run: |
+          scripts/install-${{ inputs.env_manager }}.sh --python-version ${{ inputs.python_version }}
+          echo $HOME/miniforge3/bin >>$GITHUB_PATH
+
+      - name: Print inputs
+        run: |
+          cat <<EOF
+          ${{ toJSON(inputs) }}
+          EOF
+
+      - name: Setup Triton, Torch and Transformers
+        uses: ./.github/actions/install-wheels
+        with:
+          gh_token: ${{ secrets.GITHUB_TOKEN }}
+          install_cmd: conda run --no-capture-output -n triton pip install
+          python_version: ${{ inputs.python_version }}
+
+      - name: Create test-triton command line
+        run: |
+          if [[ -n "${{ inputs.skip_list }}" ]]; then
+            skiplist="$GITHUB_WORKSPACE/scripts/skiplist/${{ inputs.skip_list }}"
+          elif [[ -n "${{ inputs.driver_version }}" ]]; then
+            skiplist="$GITHUB_WORKSPACE/scripts/skiplist/${{ inputs.driver_version }}"
+          else
+            skiplist="$GITHUB_WORKSPACE/scripts/skiplist/default"
+          fi
+
+          if [ -d "$skiplist" ]; then
+            skiplist="--skip-list $skiplist"
+          else
+            skiplist=
+          fi
+
+          {
+            echo SKIPLIST="$skiplist"
+            echo TRITON_TEST_CMD="bash -v -x scripts/test-triton.sh --warning-reports --skip-pytorch-install --reports-dir $GITHUB_WORKSPACE/reports ${{ inputs.ignore_errors && '--ignore-errors' || '' }} $skiplist"
+          } | tee -a $GITHUB_ENV
+
+      - name: Run core tests
+        run: |
+          ${{ env.TRITON_TEST_CMD }} --core
+
+      - name: Run interpreter tests
+        run: |
+          ${{ env.TRITON_TEST_CMD }} --interpreter --skip-pip-install
+
+      - name: Run Tutorials
+        run: |
+          ${{ env.TRITON_TEST_CMD }} --tutorial --skip-pip-install
+
+      - name: Run inductor test
+        run: |
+          ${{ env.TRITON_TEST_CMD }} --inductor --skip-pip-install
+
+      - name: Save conda cache
+        if: steps.conda-cache.outputs.status == 'miss'
+        uses: ./.github/actions/save
+        with:
+          path: ${{ steps.conda-cache.outputs.path }}
+          dest: ${{ steps.conda-cache.outputs.dest }}
+
+      - name: Pass rate
+        run: |
+          pip install defusedxml
+          source ./scripts/capture-hw-details.sh
+          python3 scripts/pass_rate.py --reports reports ${{ env.SKIPLIST }}
+          python3 scripts/pass_rate.py --reports reports --json ${{ env.SKIPLIST }} > pass_rate.json
+          python3 scripts/pass_rate.py --reports reports --suite tutorials --json ${{ env.SKIPLIST }} > pass_rate_tutorials.json
+
+      - name: Report environment details
+        run: |
+          source ./scripts/capture-hw-details.sh --quiet
+          cat <<EOF | tee .env
+          TIMESTAMP=$(date '+%Y%m%d%H%M%S')
+          GITHUB_RUN_ID=$GITHUB_RUN_ID
+          GITHUB_RUN_NUMBER=$GITHUB_RUN_NUMBER
+          GITHUB_RUN_ATTEMPT=$GITHUB_RUN_ATTEMPT
+          PYTHON_VERSION=${{ inputs.python_version }}
+          PYTORCH_REPO=$PYTORCH_REPO
+          PYTORCH_COMMIT_ID=$PYTORCH_COMMIT_ID
+          PYTORCH_VERSION=$PYTORCH_VERSION
+          TRITON_REPO=$GITHUB_REPOSITORY
+          LIBIGC1_VERSION=$LIBIGC1_VERSION
+          LEVEL_ZERO_VERSION=$LEVEL_ZERO_VERSION
+          GPU_DEVICE=$GPU_DEVICE
+          AGAMA_VERSION=$AGAMA_VERSION
+          EOF
+
+      - name: Upload pass rate report
+        # upload reports only for the default branch
+        if: github.ref_name == 'main'
+        uses: actions/upload-artifact@v4
+        with:
+          name: pass_rate-${{ inputs.python_version }}-${{ inputs.runner_label || inputs.driver_version }}
+          path: pass_rate*.json
+
+      - name: Upload tutorials performance report
+        uses: actions/upload-artifact@v4
+        with:
+          name: tutorials-${{ inputs.python_version }}-${{ inputs.runner_label || inputs.driver_version }}
+          if-no-files-found: warn
+          include-hidden-files: true
+          path: |
+            reports/*/*.csv
+            .env
+
+      - name: Upload test reports
+        if: inputs.upload_test_reports
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-reports-${{ inputs.python_version }}-${{ inputs.runner_label || inputs.driver_version }}
+          path: reports
diff --git a/.github/workflows/conda-build-test.yml b/.github/workflows/conda-test.yml
similarity index 97%
rename from .github/workflows/conda-build-test.yml
rename to .github/workflows/conda-test.yml
index 1b73630593..b5b81838db 100644
--- a/.github/workflows/conda-build-test.yml
+++ b/.github/workflows/conda-test.yml
@@ -74,7 +74,7 @@ jobs:
     strategy:
       matrix: ${{ fromJson(needs.prepare.outputs.matrix) }}
 
-    uses: ./.github/workflows/build-test-reusable.yml
+    uses: ./.github/workflows/conda-test-reusable.yml
     with:
       driver_version: ${{ matrix.driver }}
       runner_label: ${{ inputs.runner_label }}

From 8d42079ddb5bd0e9bcd070e5035e6785eb8a12ed Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Fri, 25 Oct 2024 01:22:08 +0200
Subject: [PATCH 102/105] Fix conda-test trigger

---
 .github/workflows/conda-test.yml | 5 ++---
 scripts/install-base.sh          | 3 ---
 scripts/run-base.sh              | 7 -------
 3 files changed, 2 insertions(+), 13 deletions(-)
 delete mode 100755 scripts/install-base.sh
 delete mode 100755 scripts/run-base.sh

diff --git a/.github/workflows/conda-test.yml b/.github/workflows/conda-test.yml
index b5b81838db..04e5342fbb 100644
--- a/.github/workflows/conda-test.yml
+++ b/.github/workflows/conda-test.yml
@@ -40,9 +40,8 @@ on:
         type: boolean
         default: false
 
-  pull_request:
-    branches:
-      - main
+  schedule:
+    - cron: "5 2 * * *"
 
 permissions: read-all
 
diff --git a/scripts/install-base.sh b/scripts/install-base.sh
deleted file mode 100755
index 64768edc7b..0000000000
--- a/scripts/install-base.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-echo Nothing to install
diff --git a/scripts/run-base.sh b/scripts/run-base.sh
deleted file mode 100755
index e3296b8d96..0000000000
--- a/scripts/run-base.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-source /opt/intel/oneapi/setvars.sh >/dev/null
-
-script_dir=$(dirname "$0")
-source "$script_dir/env-util.sh"
-source "$script_name"

From fd0ad3565a63842846694a32a549525ac6cb772a Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 28 Oct 2024 15:02:53 +0100
Subject: [PATCH 103/105] Restored TODO

---
 scripts/test-triton.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/test-triton.sh b/scripts/test-triton.sh
index 24a8bc714a..9d38b4fa35 100755
--- a/scripts/test-triton.sh
+++ b/scripts/test-triton.sh
@@ -321,6 +321,7 @@ run_inductor_tests() {
 
   pip install pyyaml pandas scipy numpy psutil pyre_extensions torchrec
 
+  # TODO: Find the fastest Hugging Face model
   ZE_AFFINITY_MASK=0 python pytorch/benchmarks/dynamo/huggingface.py --accuracy --float32 -dxpu -n10 --no-skip --dashboard --inference --freezing --total-partitions 1 --partition-id 0 --only AlbertForMaskedLM --backend=inductor --timeout=4800 --output=$(pwd -P)/inductor_log.csv
 
   cat inductor_log.csv

From 2b3e977212ed3db2bd67aee8c65caabfe97b1756 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Mon, 28 Oct 2024 15:58:37 +0100
Subject: [PATCH 104/105] Remove instrumentation path change

---
 scripts/test-triton.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/test-triton.sh b/scripts/test-triton.sh
index 9d38b4fa35..9fb2256498 100755
--- a/scripts/test-triton.sh
+++ b/scripts/test-triton.sh
@@ -302,7 +302,7 @@ run_instrumentation_tests() {
     return
   fi
 
-  INSTRUMENTATION_LIB_DIR=$(ls -1d $TRITON_PROJ/python/build/*lib*/triton/instrumentation) || SHARED_LIB_DIR=$(ls -1d $TRITON_PROJ/python/triton/_C) || err "Could not find $TRITON_PROJ/python/build/*lib*/triton/instrumentation, build Triton first"
+  INSTRUMENTATION_LIB_DIR=$(ls -1d $TRITON_PROJ/python/build/*lib*/triton/instrumentation) || err "Could not find $TRITON_PROJ/python/build/*lib*/triton/instrumentation, build Triton first"
 
   cd $TRITON_PROJ/python/test/unit
 

From 6b89eeb59a4636eafce256e714a0091f64a6ff70 Mon Sep 17 00:00:00 2001
From: Alexei Fedotov <alexei.fedotov@gmail.com>
Date: Tue, 29 Oct 2024 15:14:33 +0100
Subject: [PATCH 105/105] Add another sycl link

---
 scripts/install-conda.sh | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/scripts/install-conda.sh b/scripts/install-conda.sh
index a00d806c06..c55cc95399 100755
--- a/scripts/install-conda.sh
+++ b/scripts/install-conda.sh
@@ -1,5 +1,11 @@
 #!/bin/bash
 
+link_sycl() {
+    mkdir -p $HOME/miniforge3/envs/triton/$1
+    ln -snf /opt/intel/oneapi/compiler/2024.1/include/sycl $HOME/miniforge3/envs/triton/$1/
+    ln -snf /opt/intel/oneapi/compiler/2024.1/include/sycl/CL $HOME/miniforge3/envs/triton/$1/
+}
+
 install_env() {
     export PATH="$HOME/miniforge3/bin:$PATH"
     conda create -n triton --override-channels -c conda-forge python=$python_version.*
@@ -7,9 +13,8 @@ install_env() {
     find /opt/intel/oneapi/ \( -name '*.so' -or -name '*.so.*' \) -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
     ln -snf /usr/include/level_zero $HOME/miniforge3/envs/triton/bin/../x86_64-conda-linux-gnu/sysroot/usr/include/level_zero
     find /usr -name libze_\* -exec cp -n {} $HOME/miniforge3/envs/triton/lib \;
-    mkdir -p $HOME/miniforge3/envs/triton/lib/python$python_version/site-packages/triton/backends/intel/include
-    ln -snf /opt/intel/oneapi/compiler/2024.1/include/sycl $HOME/miniforge3/envs/triton/lib/python$python_version/site-packages/triton/backends/intel/include
-    ln -snf /opt/intel/oneapi/compiler/2024.1/include/sycl/CL $HOME/miniforge3/envs/triton/lib/python$python_version/site-packages/triton/backends/intel/include
+    link_sycl lib/python$python_version/site-packages/triton/backends/intel/include
+    link_sycl x86_64-conda-linux-gnu/sysroot/usr/include
 }
 
 print_env_info() {