Feat/model unittest ci action

.github/workflows/ce-build-ci-workflow.yml

@@ -20,7 +20,7 @@ jobs:
     with:
       flag_build: test
       runner: ernie-8gpu-2
-      image_base: ${image_base}
+      image_base: ${{env.image_base}}
 
   test-ci-images:
     name: test-ci-images
@@ -49,7 +49,7 @@ jobs:
     with:
       flag_build: update
       runner: ernie-8gpu-1
-      image_base: ${image_base}
+      image_base: ${{env.image_base}}
 
   update-ci-images-2:
     name: update-ci-images-2
@@ -58,4 +58,4 @@ jobs:
     with:
       flag_build: update
       runner: ernie-8gpu-2
-      image_base: ${image_base}
+      image_base: ${{env.image_base}}

.github/workflows/ce-unittest-gpu.yml

@@ -102,7 +102,7 @@ jobs:
           pip config set global.cache-dir "/home/.cache/pip"
           set -e
           cd /workspace/PaddleFormers && git config --global --add safe.directory $PWD
-          timeout 40m bash scripts/unit_test/ci_unit.sh ${paddle_whl} true
+          timeout 40m bash scripts/unit_test/ci_unittest.sh ${paddle_whl} true
           '
 
       - name: Upload Allure-reports & Logs

.github/workflows/debug-unittest-gpu.yml

@@ -96,6 +96,6 @@ jobs:
           echo "work_path: $work_dir/PaddleFormers"
           echo "work_path in docker: /workspace/PaddleFormers"
           echo "cmd: "
-          echo "bash scripts/unit_test/ci_unit.sh ${paddle_whl}"
+          echo "bash scripts/unit_test/ci_unittest.sh ${paddle_whl}"
           echo "or python -m pytest fail_case_name"
           echo "docker rm -f $container_name"

.github/workflows/model-unittest-gpu.yml

@@ -0,0 +1,251 @@
+name: Model Unittest GPU CI
+
+on:
+  pull_request:
+  schedule:
+    - cron: "0 18 * * *"
+  workflow_call:
+    inputs:
+      runner:
+        required: false
+        type: string
+      image_name:
+        required: false
+        type: string
+
+concurrency:
+  group: model-unittest-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  PR_ID: ${{ github.event.pull_request.number }}
+  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
+  TASK: PaddleFormers-CI-${{ github.event.pull_request.number }}-model-unittest-gpu
+  ci_scripts: /workspace/PaddleFormers/scripts/unit_test
+  BRANCH: ${{ github.event.pull_request.base.ref }}
+  AGILE_COMPILE_BRANCH: ${{ github.event.pull_request.base.ref }}
+  CI_name: model-unittest-gpu-ci
+  no_proxy: "localhost,bj.bcebos.com,su.bcebos.com,bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  model-unittest-gpu-ci:
+    name: model-unittest-gpu-ci
+    runs-on: ${{ inputs.runner || 'ernie-8gpu' }}
+    steps:
+      - name: Determine Image Name
+        env:
+          IMAGE_NAME: ${{ inputs.image_name }}
+        run: |
+          if [[ -n "${IMAGE_NAME}" ]]; then
+            echo "IMAGE_NAME=${IMAGE_NAME}" >> "$GITHUB_ENV"
+          else
+            echo "IMAGE_NAME=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda126-dev-latest" >> "$GITHUB_ENV"
+          fi
+
+      - name: Run Container
+        env:
+          work_dir: ${{ github.workspace }}
+          FLAGS_dynamic_static_unified_comm: "True"
+          python_version: "3.10"
+          paddle_whl: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuAll-LinuxCentos-Gcc11-Cuda126-Cudnn95-Trt105-Py310-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
+        run: |
+          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
+          echo "container_name=${container_name}" >> "$GITHUB_ENV"
+          docker run -d -t --gpus all -it --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
+            -v $work_dir/../../..:$work_dir/../../.. \
+            -v $work_dir:/workspace \
+            -v /home/.cache/pip:/home/.cache/pip \
+            -e BRANCH \
+            -e AGILE_COMPILE_BRANCH \
+            -e PR_ID \
+            -e COMMIT_ID \
+            -e work_dir \
+            -e ci_scripts \
+            -e no_proxy \
+            -e CI_name \
+            -e paddle_whl \
+            -e FLAGS_dynamic_static_unified_comm \
+            -e python_version \
+            -e HF_PROXY_PATH=$work_dir/../../../proxy_huggingface \
+            -e AISTUDIO_PROXY_PATH=$work_dir/../../../proxy_aistudio \
+            -e "HF_DATASETS_CACHE=$work_dir/../../../paddlenlp/huggingface/datasets" \
+            -e "TRANSFORMERS_CACHE=$work_dir/../../../paddlenlp/huggingface" \
+            -w /workspace --runtime=nvidia --privileged $IMAGE_NAME
+
+      - name: Download Code
+        run: |
+          docker exec -t $container_name /bin/bash -c '
+            rm -rf * .[^.]*
+            echo "Downloading PaddleFormers.tar"
+            wget -q --no-proxy  https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleFormers.tar --no-check-certificate
+            echo "Extracting PaddleFormers.tar"
+            tar xf PaddleFormers.tar && rm -rf PaddleFormers.tar
+            echo "work_dir = ${work_dir}"
+            source $work_dir/../../../proxy
+            cat $work_dir/../../../proxy
+            cd PaddleFormers
+            git config --global user.name "PaddleCI"
+            git config --global user.email "[email protected]"
+            git pull
+            git submodule update --init --recursive --force
+            if [ -n "${PR_ID}" ]; then
+              git fetch origin pull/${PR_ID}/head
+              git checkout -b PR_${PR_ID} FETCH_HEAD
+              git remote add upstream https://github.com/PaddlePaddle/PaddleFormers.git
+              git fetch upstream ${BRANCH}:${BRANCH}
+              git merge ${BRANCH} --no-edit
+              git diff --numstat ${BRANCH} -- | awk "{print \$NF}"
+            else
+              echo "Not in a pull_request event. Skipping PR-specific operations."
+            fi
+            git log --pretty=oneline -10
+            '
+
+      - name: Test
+        run: |
+          docker exec -t $container_name /bin/bash -c '
+          ldconfig
+          pip config set global.cache-dir "/home/.cache/pip"
+          set -e
+          rm -rf /root/.cache/aistudio/
+          cd /workspace/PaddleFormers && git config --global --add safe.directory $PWD
+          echo "work_dir = ${work_dir}"
+          cp -r $work_dir/../../../models ./models
+          echo "Check models:"
+          ls -l ./models
+          echo "Test Start"
+          hostname
+          timeout 30m bash scripts/regression/ci_model_unittest.sh ${paddle_whl}
+          '
+
+      - name: Upload Products
+        if: always()
+        env:
+          home_path: ${{ github.workspace }}/../../..
+          bos_file: ${{ github.workspace }}/../../../bos/BosClient.py
+          allure_file: ${{ github.workspace }}/../../../allure-2.19.0/bin/allure
+        run: |
+          docker exec -t $container_name /bin/bash -c '
+          if [ ! -f "${{ env.bos_file }}" ]; then
+            wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
+            mkdir ${{ env.home_path }}/bos
+            tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
+          fi
+          if [ ! -f "${{ env.allure_file }}" ]; then
+            wget -q --no-proxy -O ${{ env.home_path }}/allure-2.19.0.zip https://xly-devops.bj.bcebos.com/tools/allure-2.19.0.zip --no-check-certificate
+            unzip -q ${{ env.home_path }}/allure-2.19.0.zip
+          fi
+          if [ -n "${PR_ID}" ]; then
+            bos_prefix="${PR_ID}/${COMMIT_ID}"
+          else
+            bos_prefix="schedule/$(date +%Y%m%d)"
+          fi
+          # coverage.xml
+          # cd /workspace/PaddleFormers
+          # python ${{ env.bos_file }} coverage.xml paddle-github-action/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs
+          # echo "cov-report: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs/coverage.xml"
+          # logs
+          cd /workspace/PaddleFormers/model_unittest_logs
+          for FILE in /workspace/PaddleFormers/model_unittest_logs/*; do
+            file=$(basename "$FILE")
+            python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs
+            echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs/$file"
+          done
+          # allure
+          # cd /workspace/PaddleFormers/
+          # ${{ env.allure_file }} generate result -o report
+          # tar -czf report.tar.gz report
+          # python ${{ env.bos_file }} report.tar.gz paddle-github-action/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs
+          # echo "report: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs/report.tar.gz"
+          '
+
+      - name: Terminate And Delete the Container
+        if: always()
+        run: |
+          docker rm -f $container_name 2>/dev/null || true
+
+  # upload-coverage:
+  #   name: upload-coverage
+  #   needs: [model-unittest-gpu-ci]
+  #   if: always()
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Checkout Code
+  #       uses: actions/checkout@v4
+  #       with:
+  #         fetch-depth: 0
+
+  #     - name: Download coverage.xml
+  #       run: |
+  #         if [ -n "${PR_ID}" ]; then
+  #           bos_prefix="${PR_ID}/${COMMIT_ID}"
+  #         else
+  #           bos_prefix="schedule/$(date +%Y%m%d)"
+  #         fi
+  #         wget -q --no-proxy \
+  #           https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs/coverage.xml \
+  #           --no-check-certificate -O coverage.xml
+
+  #     - name: Fix coverage.xml paths
+  #       run: |
+  #         echo "Before fix:"
+  #         head -n 10 coverage.xml || true
+
+  #         old_source=$(grep -oPm1 '(?<=<source>).*?(?=</source>)' coverage.xml || true)
+  #         if [ -n "$old_source" ]; then
+  #           echo "Replacing source '$old_source' with 'paddleformers'"
+  #           sed -i "s|<source>$old_source</source>|<source>paddleformers</source>|g" coverage.xml
+  #         else
+  #           echo "No <source> found, injecting <source>paddleformers</source>"
+  #           sed -i 's|<sources>|<sources>\n        <source>paddleformers</source>|' coverage.xml
+  #         fi
+
+  #         echo "After fix:"
+  #         head -n 10 coverage.xml || true
+
+  #     - name: Upload coverage to Codecov
+  #       uses: codecov/codecov-action@v4
+  #       with:
+  #         files: coverage.xml
+  #       env:
+  #         CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  # upload-allure:
+  #   name: upload-allure
+  #   needs: [model-unittest-gpu-ci]
+  #   if: success() || failure()
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Checkout Code
+  #       uses: actions/checkout@v4
+  #       with:
+  #         fetch-depth: 0
+
+  #     - name: Download report.tar.gz
+  #       run: |
+  #         if [ -n "${PR_ID}" ]; then
+  #           bos_prefix="${PR_ID}/${COMMIT_ID}"
+  #         else
+  #           bos_prefix="schedule/$(date +%Y%m%d)"
+  #         fi
+  #         wget -q --no-proxy \
+  #           https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs/report.tar.gz \
+  #           --no-check-certificate -O report.tar.gz
+  #         tar -xzf report.tar.gz
+
+  #     - name: Upload Allure Report
+  #       uses: actions/upload-artifact@v4
+  #       with:
+  #         name: allure-report
+  #         path: report
+  #         if-no-files-found: ignore
+
+  #     - name: Deploy allure report to GitHub Pages
+  #       uses: peaceiris/actions-gh-pages@v4
+  #       with:
+  #         github_token: ${{ secrets.GITHUB_TOKEN }}
+  #         publish_dir: ./report

.github/workflows/unittest-gpu.yml

@@ -2,7 +2,6 @@ name: Unittest GPU CI
 
 on:
   pull_request:
-  push:
   schedule:
     - cron: "0 18 * * *"
   workflow_call:
@@ -56,6 +55,7 @@ jobs:
         run: |
           container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
           echo "container_name=${container_name}" >> "$GITHUB_ENV"
+          echo "Workspace path: ${{ github.workspace }}"
           docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
             -v $work_dir/../../..:$work_dir/../../.. \
             -v $work_dir:/workspace \
@@ -73,6 +73,8 @@ jobs:
             -e python_version \
             -e HF_PROXY_PATH=$work_dir/../../../proxy_huggingface \
             -e AISTUDIO_PROXY_PATH=$work_dir/../../../proxy_aistudio \
+            -e "HF_DATASETS_CACHE=$work_dir/../../../paddlenlp/huggingface/datasets" \
+            -e "TRANSFORMERS_CACHE=$work_dir/../../../paddlenlp/huggingface" \
             -w /workspace --runtime=nvidia --privileged $IMAGE_NAME
 
       - name: Download Code
@@ -112,7 +114,7 @@ jobs:
           cd /workspace/PaddleFormers && git config --global --add safe.directory $PWD
           source $work_dir/../../../proxy
           source $work_dir/../../../AISTUDIO_ACCESS_TOKEN
-          timeout 30m bash scripts/unit_test/ci_unit.sh ${paddle_whl}
+          timeout 30m bash scripts/unit_test/ci_unittest.sh ${paddle_whl}
           '
 
       - name: Upload Products

Makefile

@@ -19,7 +19,7 @@ format:
 
 .PHONY: lint
 lint:
-	$(eval modified_py_files := $(shell python scripts/get_modified_files.py $(check_dirs)))
+	$(eval modified_py_files := $(shell python scripts/codestyle/get_modified_files.py $(check_dirs)))
 	@if test -n "$(modified_py_files)"; then \
 		echo ${modified_py_files}; \
 		pre-commit run --files ${modified_py_files}; \

pyproject.toml

@@ -6,7 +6,6 @@ skip = ['paddleformers/transformers/__init__.py']
 [tool.black]
 line-length = 119
 target_version = ['py35', 'py36', 'py37', 'py38', 'py39', 'py310']
-exclude = ['.flake8']
 
 [tool.pytest.ini_options]
 minversion = "6.0"