Merge branch 'main' into check-tests

Author: nikita-savelyevv

.github/workflows/build_documentation.yml

@@ -0,0 +1,65 @@
+name: Build documentation
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - doc-builder*
+      - v*-release
+
+env:
+  UV_SYSTEM_PYTHON: 1
+  UV_TORCH_BACKEND: auto
+
+jobs:
+  build_documentation:
+    runs-on: ubuntu-22.04
+    env:
+      COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
+      PR_NUMBER: ${{ github.event.number }}
+      EVENT_CONTEXT: ${{ toJSON(github.event) }}
+      PR_CLONE_URL: ${{ github.event.pull_request.head.repo.clone_url }}
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "18"
+          cache-dependency-path: "kit/package-lock.json"
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Set environment variables
+        run: |
+          cd optimum
+          version=`echo "$(grep '^__version__ =' intel/version.py | cut -d '=' -f 2 | xargs)"`
+
+          if [[ $version == *.dev0 ]]
+          then
+            echo "VERSION=main" >> $GITHUB_ENV
+          else
+            echo "VERSION=v$version" >> $GITHUB_ENV
+          fi
+
+          cd ..
+
+      - name: Setup environment
+        run: |
+          pip install --upgrade pip uv
+          uv pip install git+https://github.com/huggingface/doc-builder
+          uv pip install .[quality] nncf openvino neural-compressor[pt]>3.4 diffusers accelerate
+
+      - name: Make documentation
+        shell: bash
+        run: |
+          make doc BUILD_DIR=./doc-build VERSION=${{ env.VERSION }}
+          mv ./doc-build/optimum.intel optimum-intel
+          doc-builder push optimum-intel \
+            --doc_build_repo_id "hf-doc-build/doc-build" \
+            --commit_msg "Updated with commit $COMMIT_SHA See: https://github.com/huggingface/optimum-intel/commit/$COMMIT_SHA" \
+            --token "${{ secrets.HF_DOC_BUILD_PUSH }}" \
+            --n_retries 5

.github/workflows/build_pr_documentation.yml

@@ -9,10 +9,13 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+env:
+  UV_SYSTEM_PYTHON: 1
+  UV_TORCH_BACKEND: auto
+
 jobs:
   build_documentation:
     runs-on: ubuntu-22.04
-
     env:
       COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
       PR_NUMBER: ${{ github.event.number }}
@@ -21,46 +24,31 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
-          repository: "huggingface/doc-builder"
-          path: doc-builder
+          node-version: "18"
+          cache-dependency-path: "kit/package-lock.json"
 
-      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
-          repository: "huggingface/optimum-intel"
-          path: optimum-intel
-
-      - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.9
+          python-version: "3.11"
 
       - name: Setup environment
         run: |
-          pip install --upgrade pip
-          pip uninstall -y doc-builder
-          cd doc-builder
-          git pull origin main
-          pip install .
-          pip install black
-          cd ..
+          pip install --upgrade pip uv
+          uv pip install git+https://github.com/huggingface/doc-builder
+          uv pip install .[quality] nncf openvino neural-compressor[pt]>3.4 diffusers accelerate
 
       - name: Make documentation
+        shell: bash
         run: |
-          cd optimum-intel
-          make doc BUILD_DIR=intel-doc-build VERSION=pr_$PR_NUMBER COMMIT_SHA_SUBPACKAGE=$COMMIT_SHA CLONE_URL=$PR_CLONE_URL
-          cd ..
-
-      - name: Save commit_sha & pr_number
-        run: |
-          cd optimum-intel
-          sudo chmod -R ugo+rwx intel-doc-build
-          cd intel-doc-build
-          sudo mv optimum.intel optimum-intel
+          make doc BUILD_DIR=./doc-build VERSION=pr_${{ env.PR_NUMBER }}
+          mv ./doc-build/optimum.intel optimum-intel
           echo ${{ env.COMMIT_SHA }} > ./commit_sha
           echo ${{ env.PR_NUMBER }} > ./pr_number
 
       - uses: actions/upload-artifact@v4
         with:
           name: doc-build-artifact
-          path: optimum-intel/intel-doc-build/
+          path: optimum-intel

.github/workflows/quality.yml

@@ -29,7 +29,7 @@ jobs:
         run: |
           pip install --upgrade pip
           # .[quality] installs too many dependencies
-          # TODO: we should remove the the version pinning at some point
+          # TODO: we should remove the version pinning at some point
           pip install "black~=23.1" "ruff==0.4.4"
 
       - name: Check style with black

.github/workflows/style_bot.yml

@@ -0,0 +1,17 @@
+name: Style Bot
+
+on:
+  issue_comment:
+    types: [created]
+
+permissions:
+  pull-requests: write
+
+jobs:
+  style:
+    uses: huggingface/huggingface_hub/.github/workflows/style-bot-action.yml@main
+    with:
+      python_quality_dependencies: "[quality]"
+      style_command_type: "style_only"
+    secrets:
+      bot_token: ${{ secrets.HF_STYLE_BOT_ACTION }}

.github/workflows/test_inc.yml

@@ -13,6 +13,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+env:
+  TRANSFORMERS_IS_CI: true
+
 jobs:
   build:
     strategy:
@@ -35,7 +38,7 @@ jobs:
         run: |
           pip install --upgrade pip
           pip install torch==${{ matrix.torch-version }} torchaudio torchvision --index-url https://download.pytorch.org/whl/cpu
-          pip install .[neural-compressor,diffusers,peft,tests] transformers[testing] intel-extension-for-pytorch==${{ matrix.torch-version }}
+          pip install .[tests,neural-compressor] intel-extension-for-pytorch==${{ matrix.torch-version }} diffusers==0.32.2
 
       - name: Assert versions
         run: |
@@ -45,3 +48,5 @@ jobs:
       - name: Test with Pytest
         run: |
           pytest tests/neural_compressor
+        env:
+          HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }}

.github/workflows/test_ipex.yml

@@ -13,13 +13,17 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+env:
+  TRANSFORMERS_IS_CI: true
+
 jobs:
   build:
     strategy:
       fail-fast: false
       matrix:
-        transformers-version: ["4.49.0"]
-        torch-version: ["2.6.0"]
+        transformers-version: ["4.51.0", "4.52.*"]
+        torch-version: ["2.7.0"]
+        test-file: [test_modeling.py, test_pipelines.py, test_modeling_causal_lm.py]
 
     runs-on: ubuntu-22.04
 
@@ -39,16 +43,19 @@ jobs:
           pip install .[ipex,tests] transformers[testing]==${{ matrix.transformers-version }} intel_extension_for_pytorch==${{ matrix.torch-version }}
 
       - name: Install bitsandbytes
+        if: matrix.test-file == 'test_modeling_causal_lm.py'
         run: |
           git clone --branch multi-backend-refactor https://github.com/bitsandbytes-foundation/bitsandbytes.git
           cd bitsandbytes
           pip install .
 
       - name: Install autoawq
+        if: matrix.test-file == 'test_modeling_causal_lm.py'
         run: |
           git clone https://github.com/casper-hansen/AutoAWQ.git
           cd AutoAWQ
           pip install .
+          pip uninstall triton -y
 
       - name: Assert versions
         run: |
@@ -58,4 +65,4 @@ jobs:
 
       - name: Test with Pytest
         run: |
-          pytest tests/ipex
+          pytest tests/ipex/${{ matrix.test-file }} -rsx -v

.github/workflows/test_offline.yaml

@@ -13,6 +13,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+env:
+  TRANSFORMERS_IS_CI: true
+
 jobs:
   build:
     runs-on: ubuntu-22.04

.github/workflows/test_openvino.yml

@@ -14,18 +14,16 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+env:
+  TRANSFORMERS_IS_CI: true
+
 jobs:
   build:
     strategy:
       fail-fast: false
       matrix:
         test-pattern:
-          [
-            "*modeling*",
-            "*diffusion*",
-            "*quantization*",
-            "*export*",
-          ]
+          ["*modeling*", "*diffusion*", "*quantization*", "*export*"]
         transformers-version: ["4.36.0", "latest"]
 
     runs-on: ubuntu-22.04
@@ -43,15 +41,15 @@ jobs:
         run: |
           pip install --upgrade pip
           pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
-          pip install .[openvino,openvino-tokenizers,diffusers,tests] transformers[testing]
+          pip install .[openvino,diffusers,tests]
           pip install --pre -U openvino openvino-tokenizers --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/pre-release
           pip install git+https://github.com/openvinotoolkit/nncf.git@release_v2180
 
       - if: ${{ matrix.transformers-version != 'latest' }}
         name: Install specific dependencies and versions required for older transformers
         run: |
           pip install transformers==${{ matrix.transformers-version }} accelerate==0.* peft==0.13.* diffusers==0.30.* transformers_stream_generator
-      
+
       - if: ${{ matrix.transformers-version == 'latest' && matrix.test-pattern == '*modeling*'}}
         name: Install auto-gptq, autoawq
         run: |
@@ -77,3 +75,5 @@ jobs:
         name: Test with Pytest (nightly)
         run: |
           pytest tests/openvino/test_modeling_basic.py --durations=0
+        env:
+          HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }}

.github/workflows/test_openvino_full.yml

@@ -14,49 +14,40 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+env:
+  TRANSFORMERS_IS_CI: true
+
 jobs:
   build:
     if: ${{ (github.event_name == 'workflow_dispatch') || (github.event_name == 'schedule') ||  (github.event_name == 'push') || contains( github.event.pull_request.labels.*.name, 'openvino-test') }}
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - python-version: "3.9"
-            os: "ubuntu-22.04"
-            transformers-version: "latest"
-            openvino: "ov-stable"
-            nncf: "nncf-stable"
-          - python-version: "3.9"
-            os: "ubuntu-22.04"
-            transformers-version: "latest"
-            openvino: "ov-nightly"
-            nncf: "nncf-stable"
-          - python-version: "3.9"
-            os: "ubuntu-22.04"
-            transformers-version: "latest"
-            openvino: "ov-stable"
-            nncf: "nncf-develop"
-          - python-version: "3.9"
-            os: "ubuntu-22.04"
-            transformers-version: "latest"
-            openvino: "ov-nightly"
-            nncf: "nncf-develop"
-
-    runs-on: ${{ matrix.os }}
+        nncf: ["nncf-stable", "nncf-develop"]
+        openvino: ["ov-stable", "ov-nightly"]
+        transformers-version: ["latest"]
+
+    runs-on: ubuntu-22.04
 
     steps:
-      - uses: actions/checkout@v4
-      - name: Setup Python ${{ matrix.python-version }}
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: 3.9
 
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          pip install --upgrade pip
           # Install PyTorch CPU to prevent unnecessary downloading/installing of CUDA packages
           pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
-          pip install .[tests]
+          pip install .[tests,diffusers]
+          pip uninstall opencv-python -y && pip install opencv-python-headless
 
       - name: Install openvino-nightly
         if: ${{ matrix.openvino == 'ov-nightly' }}
@@ -78,7 +69,7 @@ jobs:
         if: ${{ matrix.transformers-version != 'latest' }}
         run: pip install transformers==${{ matrix.transformers-version }}
 
-      - if: ${{ matrix.transformers-version == 'latest' && matrix.os != 'windows-2019' }}
+      - if: ${{ matrix.transformers-version == 'latest' }}
         name: Install auto-gptq, autoawq
         run: |
           pip install auto-gptq "autoawq<0.2.8" --extra-index-url https://download.pytorch.org/whl/cpu