[CE]fix ce scripts (#2534)

Author: Liujie0926

.github/workflows/ce-build.yml

@@ -4,15 +4,10 @@ on:
   push:
     branches:
       - develop
-      
-concurrency:
-  group: build-${{ github.workflow }}
-  cancel-in-progress: true
 
 env:
-  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
-  TASK: PaddleFormers-CE-${{ github.event.pull_request.number }}-build
-  BRANCH: ${{ github.event.pull_request.base.ref }}
+  COMMIT_ID: ${{ github.sha }}
+  TASK: PaddleFormers-CE-${{ github.sha }}-build
   CE_name: build-ce
   no_proxy: "localhost,bj.bcebos.com,su.bcebos.com,bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
 
@@ -23,7 +18,7 @@ defaults:
 jobs:
   build-ce:
     name: build-ce
-    runs-on: [self-hosted, ernie-8gpu]
+    runs-on: [self-hosted, paddleformers]
     steps:
       - name: Determine Image Name
         run: |
@@ -43,7 +38,6 @@ jobs:
             -v $work_dir:/workspace \
             -v /home/.cache/pip:/home/.cache/pip \
             -e BRANCH \
-            -e AGILE_COMPILE_BRANCH \
             -e COMMIT_ID \
             -e work_dir \
             -e ce_scripts \
@@ -68,6 +62,7 @@ jobs:
           git config --global user.email "[email protected]"
           git pull
           git submodule update --init --recursive --force
+          git checkout -b $COMMIT_ID $COMMIT_ID
           git log --pretty=oneline -10
           '
 
@@ -83,25 +78,23 @@ jobs:
           bash scripts/regression/build.sh
           '
           
-      - name: Upload Logs
+      - name: Upload Products
         if: always()
         env:
           home_path: ${{ github.workspace }}/../../..
           bos_file: ${{ github.workspace }}/../../../bos/BosClient.py
-          allure_file: ${{ github.workspace }}/../../../allure-2.19.0/bin/allure
         run: |
           docker exec -t $container_name /bin/bash -c '
           if [ ! -f "${{ env.bos_file }}" ]; then
             wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
             mkdir ${{ env.home_path }}/bos
             tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
           fi
-          bos_prefix="schedule/$(date +%Y%m%d)"
-          cd /workspace/PaddleFormers/build_logs
-          for FILE in /workspace/PaddleFormers/build_logs/*; do
+          cd /workspace/PaddleFormers/upload
+          for FILE in /workspace/PaddleFormers/upload/*; do
             file=$(basename "$FILE")
-            python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/build/${bos_prefix}/logs
-            echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/build/${bos_prefix}/logs/$file"
+            python ${{ env.bos_file }} $file paddleformers/wheels
+            echo "$file: https://paddleformers.bj.bcebos.com/wheels/$file"
           done
           '
 

.github/workflows/ce-deadlink.yml

@@ -24,7 +24,7 @@ jobs:
   deadlink-ce:
     name: deadlink-ce
     if: github.ref == 'refs/heads/develop'
-    runs-on: [self-hosted, ernie-8gpu]
+    runs-on: [self-hosted, paddleformers]
     steps:
       - name: Determine Image Name
         run: |
@@ -44,7 +44,6 @@ jobs:
             -v $work_dir:/workspace \
             -v /home/.cache/pip:/home/.cache/pip \
             -e BRANCH \
-            -e AGILE_COMPILE_BRANCH \
             -e COMMIT_ID \
             -e work_dir \
             -e ce_scripts \
@@ -86,6 +85,7 @@ jobs:
           pip config set global.cache-dir "/home/.cache/pip"
           set -e
           python -m pip install beautifulsoup4 openpyxl
+          source $work_dir/../../../proxy
           bash run.sh PaddleFormers develop [email protected] PaddleFormers死链检测汇总报告
           '
           
@@ -94,7 +94,6 @@ jobs:
         env:
           home_path: ${{ github.workspace }}/../../..
           bos_file: ${{ github.workspace }}/../../../bos/BosClient.py
-          allure_file: ${{ github.workspace }}/../../../allure-2.19.0/bin/allure
         run: |
           docker exec -t $container_name /bin/bash -c '
           if [ ! -f "${{ env.bos_file }}" ]; then
@@ -103,8 +102,8 @@ jobs:
             tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
           fi
           bos_prefix="schedule/$(date +%Y%m%d)"
-          cd /workspace/PaddleFormers/deadlink_logs
-          for FILE in /workspace/PaddleFormers/deadlink_logs/*; do
+          cd /workspace/PaddleFormers/result
+          for FILE in /workspace/PaddleFormers/result/*; do
             file=$(basename "$FILE")
             python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/deadlink/${bos_prefix}/logs
             echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/deadlink/${bos_prefix}/logs/$file"

.github/workflows/ce-unittest-gpu.yml

@@ -4,6 +4,16 @@ on:
   schedule:
     - cron: "0 6 * * 6"
   workflow_dispatch:        # allow to manually trigger the workflow
+    inputs:
+      paddle_whl:
+        description: "paddle链接"
+        required: false
+        default: "https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CINN-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl"
+        type: string
+      repo_branch:
+        description: "套件分支"
+        required: false
+        type: string
 
 concurrency:
   group: unittest-${{ github.workflow }}
@@ -37,7 +47,8 @@ jobs:
           work_dir: ${{ github.workspace }}
           FLAGS_dynamic_static_unified_comm: "True"
           python_version: "3.10"
-          paddle_whl: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CENN-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
+          paddle_whl: ${{ github.event.inputs.paddle_whl || 'https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CINN-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl' }}
+          repo_branch: ${{ github.event.inputs.repo_branch || '' }}
         run: |
           container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
           echo "container_name=${container_name}" >> "$GITHUB_ENV"
@@ -53,6 +64,7 @@ jobs:
             -e no_proxy \
             -e CE_name \
             -e paddle_whl \
+            -e repo_branch \
             -e FLAGS_dynamic_static_unified_comm \
             -e python_version \
             -w /workspace --runtime=nvidia $IMAGE_NAME
@@ -69,7 +81,15 @@ jobs:
           cd PaddleFormers
           git config --global user.name "PaddleCE"
           git config --global user.email "[email protected]"
-          git pull
+          if [ -n "$repo_branch" ]; then
+            echo "Switching to branch: $repo_branch"
+            git fetch origin $repo_branch
+            git checkout $repo_branch || git checkout -b $repo_branch origin/$repo_branch
+            git pull
+          else
+            echo "No repo_branch provided, just pulling latest changes"
+            git pull
+          fi
           git submodule update --init --recursive --force
           git log --pretty=oneline -10
           '

scripts/regression/build.sh

@@ -18,8 +18,8 @@ set -e
 export formers_dir=/workspace/PaddleFormers
 mkdir -p /workspace/PaddleFormers/build_logs
 export log_path=/workspace/PaddleFormers/build_logs
-mkdir -p ${PPNLP_HOME}/upload_${AGILE_PIPELINE_BUILD_NUMBER}
-upload_path=${PPNLP_HOME}/upload_${AGILE_PIPELINE_BUILD_NUMBER}
+mkdir -p /workspace/PaddleFormers/upload
+upload_path=/workspace/PaddleFormers/upload
 export Build_list=()
 
 python -m pip config --user set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
@@ -91,19 +91,17 @@ Build_list=($(awk -v RS=' ' '!a[$1]++' <<< ${Build_list[*]}))
 if [[ ${#Build_list[*]} -ne 0 ]];then
     echo -e "\033[31m ---- Build_list length: ${#Build_list[*]}, cases: ${Build_list[*]} \033[0m"
     echo -e "\033[31m ============================= \033[0m"
-    install_paddle
+    # install_paddle
     if [[ $(contain_case paddleformers ${Build_list[@]}; echo $?) -eq 1 ]];then
         paddleformers_build
-    else
-        install_paddleformers
     fi
 
-    if [ -e "${upload_path}" ] && [ "$(ls -A "${upload_path}/")" ]; then
-        cd ${upload_path} && ls -A "${upload_path}"
-        cd ${PPNLP_HOME} && python upload.py ${upload_path} 'paddleformers/wheels'
-        rm -rf ${upload_path}
-        echo -e "\033[32m upload wheels SUCCESS \033[0m"
-    fi
+    # if [ -e "${upload_path}" ] && [ "$(ls -A "${upload_path}/")" ]; then
+    #     cd ${upload_path} && ls -A "${upload_path}"
+    #     python /workspace/../../../bos/BosClient.py ${upload_path} 'paddleformers/wheels'
+    #     rm -rf ${upload_path}
+    #     echo -e "\033[32m upload wheels SUCCESS \033[0m"
+    # fi
 else
     echo -e "\033[32m Don't need build any whl  \033[0m"
 fi