Optimize CPU CI pipelines (#1071)

Signed-off-by: Sun, Xuehao <[email protected]>

Author: XuehaoSun

.azure-pipelines/docker/Dockerfile.devel

@@ -19,11 +19,8 @@ FROM ubuntu:${UBUNTU_VER}
 ENV LANG C.UTF-8
 
 RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
-    python3 \
-    python3-pip \
-    python3.12-dev \
-    autoconf \
     build-essential \
+    ca-certificates \
     git \
     libomp-dev \
     numactl \
@@ -33,7 +30,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
     jq \
     vim
 
-RUN ln -sf $(which python3) /usr/bin/python
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
 
 ARG USER_ID=1000
 ARG GROUP_ID=1000
@@ -43,10 +40,13 @@ RUN groupadd -g ${GROUP_ID} hostgroup && \
 
 USER hostuser
 
-ENV PATH="/home/hostuser/.local/bin:$PATH"
-RUN pip config set global.break-system-packages true
-RUN python -m pip install --no-cache-dir --upgrade pip setuptools
-RUN pip list
+ENV PATH="/home/hostuser/.venv/bin:$PATH"
+ENV VIRTUAL_ENV="/home/hostuser/.venv"
+ENV UV_NO_PROGRESS=1 \
+    UV_COMPILE_BYTECODE=1 \
+    UV_LINK_MODE=copy
 
-WORKDIR /
+RUN uv venv --python=3.12 /home/hostuser/.venv
+RUN which python && python --version
 
+WORKDIR /home/hostuser

.azure-pipelines/scripts/ut/.coverage

@@ -4,6 +4,7 @@ branch = True
 [paths]
 source =
     auto_round/
+    /auto-round/auto_round/
     */site-packages/auto_round/
     */dist-packages/auto_round/
 

.azure-pipelines/scripts/ut/collect_log.sh

@@ -1,18 +1,18 @@
 #!/bin/bash
 set -e
-pip install coverage
-export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coverage
-coverage_log="/auto-round/log_dir/coverage_log"
-cd /auto-round/log_dir
+uv pip install coverage
+export COVERAGE_RCFILE=${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coverage
+coverage_log="${BUILD_SOURCESDIRECTORY}/log_dir/coverage_log"
+cd "${BUILD_SOURCESDIRECTORY}/log_dir"
 
 echo "collect coverage for PR branch"
 mkdir -p coverage_PR
 cp ut-*/.coverage.* ./coverage_PR/
 cd coverage_PR
 coverage combine --keep --rcfile=${COVERAGE_RCFILE}
 
-cp .coverage /auto-round/
-cd /auto-round
+cp .coverage "${BUILD_SOURCESDIRECTORY}"
+cd "${BUILD_SOURCESDIRECTORY}"
 coverage report -m --rcfile=${COVERAGE_RCFILE} | tee ${coverage_log}
 coverage html -d log_dir/coverage_PR/htmlcov --rcfile=${COVERAGE_RCFILE}
 coverage xml -o log_dir/coverage_PR/coverage.xml --rcfile=${COVERAGE_RCFILE}

.azure-pipelines/scripts/ut/run_ut.sh

@@ -6,18 +6,24 @@ test_part=$1
 # install requirements
 echo "##[group]set up UT env..."
 export TQDM_MININTERVAL=60
-pip install pytest-cov pytest-html
-pip install -r /auto-round/test/test_cpu/requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu
-pip list
+uv pip install pytest-cov pytest-html
+uv pip install -r /auto-round/test/test_cpu/requirements.txt \
+    --extra-index-url https://download.pytorch.org/whl/cpu
+
 # install latest gguf for ut test
-git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py && pip install .
+cd ~ || exit 1
+git clone -b master --quiet --single-branch https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py && uv pip install .
+
+cd /auto-round && uv pip install .
+uv pip install lm_eval==0.4.9.1
+
 echo "##[endgroup]"
-pip list
+uv pip list
 
 cd /auto-round/test/test_cpu || exit 1
 find . -type f -exec sed -i '/sys\.path\.insert(0, "\.\.")/d' {} +
 
-export LD_LIBRARY_PATH=${HOME}/.local/lib/:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=${HOME}/.venv/lib/:$LD_LIBRARY_PATH
 export FORCE_BF16=1
 export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coverage
 auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')

.azure-pipelines/template/docker-template.yml

@@ -40,6 +40,7 @@ steps:
 
       - checkout: self
         clean: true
+        fetchTags: true
         displayName: "Checkout out Repo"
 
   - ${{ if eq(parameters.dockerConfigName, 'gitCloneDockerConfig') }}:

.azure-pipelines/template/ut-template.yml

@@ -35,12 +35,11 @@ steps:
   - ${{ if eq(parameters.imageSource, 'build') }}:
     - script: |
         docker exec ${{ parameters.utContainerName }} bash -c "cd /auto-round \
-          && pip install torch==2.8.0 torchvision --index-url https://download.pytorch.org/whl/cpu \
-          && pip install intel-extension-for-pytorch==2.8.0 \
-          && pip install -r requirements-cpu.txt \
-          && pip install transformers==4.57.1 \
-          && pip install . \
-          && pip list"
+          && uv pip install torch==2.8.0 torchvision --index-url https://download.pytorch.org/whl/cpu \
+          && uv pip install intel-extension-for-pytorch==2.8.0 \
+          && uv pip install -r requirements.txt \
+          && uv pip install -r requirements-cpu.txt \
+          && uv pip list"
       displayName: "Env Setup"
 
   - ${{ if eq(parameters.imageSource, 'pull') }}:

.azure-pipelines/unit-test-hpu.yml

@@ -26,7 +26,7 @@ pool: GAUDI
 
 variables:
   IMAGE_NAME: "auto-round"
-  IMAGE_TAG: "py310"
+  IMAGE_TAG: "py312"
   UPLOAD_PATH: $(Build.SourcesDirectory)/log_dir
   DOWNLOAD_PATH: $(Build.SourcesDirectory)/log_dir
   ARTIFACT_NAME: "UT_coverage_report"
@@ -48,11 +48,6 @@ stages:
               uploadPath: $(UPLOAD_PATH)
               utArtifact: "ut"
 
-          - task: UseDotNet@2
-            displayName: 'Use .NET Core sdk 7.0.x'
-            inputs:
-              version: 7.0.x
-
           - task: PublishCodeCoverageResults@2
             inputs:
               summaryFileLocation: $(UPLOAD_PATH)/coverage.xml

.azure-pipelines/unit-test.yml

@@ -27,7 +27,7 @@ pool: ICX-16C
 
 variables:
   IMAGE_NAME: "auto-round"
-  IMAGE_TAG: "py310"
+  IMAGE_TAG: "py312"
   UPLOAD_PATH: $(Build.SourcesDirectory)/log_dir
   DOWNLOAD_PATH: $(Build.SourcesDirectory)/log_dir
   ARTIFACT_NAME: "UT_coverage_report"
@@ -69,33 +69,29 @@ stages:
     jobs:
       - job: CollectDatafiles
         steps:
-          - script: |
-              docker build --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} .
-              docker images | grep -i ${IMAGE_NAME}
-              if [[ $? -ne 0 ]]; then
-                echo "NO Such Repo"
-                exit 1
-              fi
-            displayName: "Build develop docker image"
-
           - task: DownloadPipelineArtifact@2
             inputs:
               artifact:
               patterns: '*_coverage/.coverage.*'
               path: $(DOWNLOAD_PATH)
+          
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: '3.12'
+            displayName: 'Use Python 3.12'
 
           - script: |
-              echo "--- create container ---"
-              docker run -d -it --name="collectLogs"  -v ${BUILD_SOURCESDIRECTORY}:/auto-round  ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash
-              echo "--- docker ps ---"
-              docker ps
-              echo "--- collect logs ---"
-              docker exec collectLogs bash -c "cd /auto-round \
-                && pip install -r requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu \
-                && pip install . \
-                && pip list"
-              docker exec collectLogs /bin/bash +x -c "cd /auto-round/.azure-pipelines/scripts \
-                && bash ut/collect_log.sh"
+              cd ${BUILD_SOURCESDIRECTORY}
+              pip install -U pip setuptools uv
+              uv pip install -r requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu
+              uv pip install .
+              pip list
+              cd ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts
+              bash ut/collect_log.sh
+            env:
+              PYTHONUNBUFFERED: '1'
+              UV_NO_PROGRESS: '1'
+              UV_SYSTEM_PYTHON: '1'
             displayName: "Collect UT Coverage"
 
           - task: PublishPipelineArtifact@1
@@ -105,11 +101,6 @@ stages:
               artifact: $(ARTIFACT_NAME)
               publishLocation: "pipeline"
 
-          - task: UseDotNet@2
-            displayName: 'Use .NET Core sdk 7.0.x'
-            inputs:
-              version: 7.0.x
-
           - task: PublishCodeCoverageResults@2
             inputs:
               summaryFileLocation: $(Build.SourcesDirectory)/log_dir/coverage_PR/coverage.xml

README.md

@@ -10,7 +10,7 @@
 <h3> Advanced Quantization Algorithm for LLMs</h3>
 
 [![python](https://img.shields.io/badge/python-3.10%2B-blue)](https://github.com/intel/auto-round)
-[![version](https://img.shields.io/badge/release-0.9.0-green)](https://github.com/intel/auto-round)
+[![version](https://img.shields.io/badge/release-0.9.1-green)](https://github.com/intel/auto-round)
 [![license](https://img.shields.io/badge/license-Apache%202-9C27B0)](https://github.com/intel/auto-round/blob/main/LICENSE)
 <a href="https://huggingface.co/Intel">
 <img alt="Model Checkpoints" src="https://img.shields.io/badge/%F0%9F%A4%97%20HF-Models-F57C00">

auto_round/version.py

@@ -14,4 +14,4 @@
 """Intel® auto-round: An open-source Python library
 supporting popular model weight only compression based on signround."""
 
-__version__ = "0.9.0"
+__version__ = "0.9.2"