Drops torch 2.8 on CI, more unit tests for patches (#323)

* update ci

* freeze

* patch

* better with cache

* shorten names

* few changes

Author: xadupre

.github/workflows/ci.yml

@@ -17,8 +17,8 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python: ['3.10', '3.11', '3.12', '3.13']
-        transformers: ['4.48.3', '4.51.3', '4.52.4', '4.55.4', '4.56.2', '4.57.1', 'main']
-        torch: ['2.8', '2.9', 'main']
+        transformers: ['4.48.3', '4.51.3', '4.52.4', '4.55.4', '4.56.2', '4.57.3', 'main']
+        torch: ['2.9', 'main']
         exclude:
           - python: '3.10'  # 3.10
             torch: 'main'
@@ -33,22 +33,18 @@ jobs:
           - python: '3.10'
             transformers: '4.56.2'
           - python: '3.10'
-            transformers: '4.57.1'
+            transformers: '4.57.3'
           - python: '3.11'  # 3.11
             torch: 'main'
-          - python: '3.11'
-            torch: '2.8'
           - python: '3.11'
             transformers: 'main'
           - python: '3.11'
             transformers: '4.55.4'
           - python: '3.11'
             transformers: '4.56.2'
           - python: '3.11'
-            transformers: '4.57.1'
-          - python: '3.13'  # 3.13
-            torch: '2.8'
-          - python: '3.13'
+            transformers: '4.57.3'
+          - python: '3.13'  # 3.11
             torch: '2.9'
           - python: '3.13'
             transformers: '4.48.3'
@@ -111,6 +107,7 @@ jobs:
           fi
 
       - name: Cache pip
+        if: ${{ matrix.torch != 'main' && matrix.transformers != 'main' }}
         uses: actions/cache@v4
         with:
           path: ~/.cache/pip

.github/workflows/documentation.yml

@@ -50,15 +50,6 @@ jobs:
           python -m pip uninstall -y onnx
           python -m pip install onnx-weekly
 
-      - name: Cache pip
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-            ${{ runner.os }}-
-
       - name: pip freeze
         run: python -m pip freeze
 

.github/workflows/models.yml

@@ -0,0 +1,65 @@
+name: MODELS
+
+on:
+  push:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - main
+
+jobs:
+  run:
+    name: to-${{ matrix.torch }}-tr-${{ matrix.transformers }}-ci ${{ matrix.os }}-${{ matrix.python }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python: ['3.12']
+        transformers: ['4.57.3']
+        torch: ['main']
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Install pytorch ${{ matrix.torch }}
+        run: |
+          if [[ "${{ matrix.torch }}" == "main" ]]; then
+            python -m pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu
+          else
+            echo "install torch==${{ matrix.torch }} torchvision torchaudio"
+            pip install torch==${{ matrix.torch }} torchvision torchaudio
+          fi
+
+      - name: Install transformers ${{ matrix.transformers }}
+        run: |
+          if [[ "${{ matrix.transformers }}" == "main" ]]; then
+            echo "install transformers from github"
+            git clone https://github.com/huggingface/transformers.git
+            cd transformers
+            pip install -e .
+            cd ..
+          else
+            echo "install transformers==${{ matrix.transformers }}"
+            pip install transformers==${{ matrix.transformers }}
+          fi
+
+      - name: Install requirements
+        run: python -m pip install -r requirements.txt
+
+      - name: Install requirements dev
+        run: python -m pip install -r requirements-dev.txt
+
+      - name: Uninstall onnx-diagnostic
+        run: python -m pip uninstall -y onnx-diagnostic
+
+      - name: pip freeze
+        run: python -m pip freeze
+
+      - name: qwen2.5_vl_instruct
+        run: PYTHONPATH=. UNITTEST_GOING=1 NEVERTEST=1 QWEN25ATTENTION=BIGMASK TESTDTYPE=float16 TESTDEVICE=cpu python _unittests/ut_tasks/try_export.py -f -k test_qwen_2_5_vli_visual
+        

CHANGELOGS.rst

@@ -4,6 +4,7 @@ Change Logs
 0.8.3
 +++++
 
+* :pr:`323`: drops torch 2.8 on CI
 * :pr:`322`: support rerunning onnx kernels with torch intermediate results in side-by-side
 * :pr:`314`: fix modelbuilder download needed after this change https://github.com/microsoft/onnxruntime-genai/pull/1862
 * :pr:`311`: use custom and local function to use PackedMultiHeadAttention from onnxruntime

_doc/cmds/sbs.rst

@@ -27,9 +27,9 @@ Example
 .. code-block::
 
     python -m onnx_diagnostic sbs \
-        -i qwen_2_5_vl_instruct_visual.inputs.pt \
-        --ep test_imagetext2text_qwen_2_5_vl_instruct_visual.cuda.float16.custom.graph.ep.pt2 \
-        -m test_imagetext2text_qwen_2_5_vl_instruct_visual.cuda.float16.custom.onnx \
+        -i qwen25_vli_visual.inputs.pt \
+        --ep test_qwen25_vli_visual.cuda.float16.custom.graph.ep.pt2 \
+        -m test_qwen25_vli_visual.cuda.float16.custom.onnx \
         -o results.dynamo.float16.xlsx \
         -v 1 --atol=0.1 --rtol=1 \
         --replay-names conv3d,rsqrt,to_4,mul_48,linear,linear_2,linear_84,linear_89,mul_172,linear_156,linear_159 \