[CI]update build.sh (#10941)

* update build.sh

* add proxy

* update cache path

* skip aistudio down for connection

* fix

Author: Liujie0926

.github/workflows/lint.yml

@@ -72,8 +72,6 @@ jobs:
           else
             echo "local develop branch exist, skipping"
           fi
-
-          unset http_proxy && unset https_proxy
           '
           
       - name: Setup Environment
@@ -84,6 +82,7 @@ jobs:
           set -e
           python -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
           python -m pip config set global.cache-dir "/home/.cache/pip"
+          source $work_dir/../../../proxy
           python -m pip install --upgrade pip
           cd /workspace/PaddleNLP && git config --global --add safe.directory $PWD
           make install
@@ -93,6 +92,7 @@ jobs:
         run: |
           docker exec -t $container_name /bin/bash -c '
           set -e
+          source $work_dir/../../../proxy
           cd /workspace/PaddleNLP
           make lint
           '

.github/workflows/unittest-cpu.yml

@@ -11,6 +11,11 @@ env:
   COMMIT_ID: ${{ github.event.pull_request.head.sha }}
   BRANCH: ${{ github.event.pull_request.base.ref }}
   TASK: PaddleNLP-CI-${{ github.event.pull_request.number }}-unittest-cpu
+  HF_ENDPOINT: https://hf-mirror.com
+  STUDIO_GIT_HOST: http://git.prod.idc-to-cloud.aistudio.baidu-int.com
+  PPNLP_HOME: /home/disk1/cache
+  HF_DATASETS_CACHE: /home/disk1/cache/huggingface/datasets
+  TRANSFORMERS_CACHE: /home/disk1/cache/huggingface
 
 jobs:
   Test:
@@ -33,12 +38,18 @@ jobs:
             -v $work_dir/../../..:$work_dir/../../.. \
             -v $work_dir:/workspace \
             -v /home/.cache/pip:/home/.cache/pip \
+            -v /home/disk1/cache:/home/disk1/cache \
             -e BRANCH \
             -e PR_ID \
             -e COMMIT_ID \
             -e work_dir \
             -e no_proxy \
             -e python_version \
+            -e HF_ENDPOINT \
+            -e STUDIO_GIT_HOST \
+            -e PPNLP_HOME \
+            -e HF_DATASETS_CACHE \
+            -e TRANSFORMERS_CACHE \
             -w /workspace ${docker_image}
 
       - name: Download Code
@@ -68,7 +79,6 @@ jobs:
             echo "Not in a pull_request event. Skipping PR-specific operations."
           fi
           git log --pretty=oneline -10
-          unset http_proxy && unset https_proxy
           '
 
       - name: Setup Environment
@@ -79,6 +89,7 @@ jobs:
           set -e
           python -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
           python -m pip config set global.cache-dir "/home/.cache/pip"
+          source $work_dir/../../../proxy
           python -m pip install --upgrade pip
           cd /workspace/PaddleNLP && git config --global --add safe.directory $PWD
           pip install -r tests/requirements.txt

scripts/regression/build.sh

@@ -22,16 +22,17 @@ mkdir -p ${PPNLP_HOME}/upload_${AGILE_PIPELINE_BUILD_NUMBER}
 upload_path=${PPNLP_HOME}/upload_${AGILE_PIPELINE_BUILD_NUMBER}
 export Build_list=()
 
-python -m pip config --user set global.index http://pip.baidu-int.com/search/
-python -m pip config --user set global.index-url http://pip.baidu-int.com/simple
-python -m pip config --user set global.trusted-host pip.baidu-int.com
+python -m pip config --user set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
+python -m pip config --user set global.trusted-host pypi.tuna.tsinghua.edu.cn
 
 get_diff_case(){
     git diff --name-only HEAD~1 HEAD
     for file_name in `git diff --name-only HEAD~1 HEAD`;do
         arr_file_name=(${file_name//// })
         if [[ ${arr_file_name[0]} == "paddlenlp" ]];then
             Build_list[${#Build_list[*]}]="paddlenlp"
+        elif [[ ${arr_file_name[0]} == "requirements"* ]]; then
+            Build_list[${#Build_list[*]}]="paddlenlp"
         elif [[ ${arr_file_name[0]} == "csrc" ]];then
             Build_list[${#Build_list[*]}]="paddlenlp_ops"
         else
@@ -64,12 +65,9 @@ paddlenlp_build (){
     rm -rf paddle_pipelines.egg-info/
     rm -rf dist/
 
-    python -m pip install -r requirements.txt --trusted-host pip.baidu-int.com
-    python -m pip install -r requirements-dev.txt --trusted-host pip.baidu-int.com
+    python -m pip install -r requirements.txt
+    python -m pip install -r requirements-dev.txt
     python setup.py bdist_wheel
-    python -m pip uninstall protobuf -y
-    python -m pip install protobuf==3.20.2
-    python -m pip install numpy==1.26.4 --force-reinstall
     python -m pip install --ignore-installed  dist/p****.whl --force-reinstall
     python -c "import paddlenlp; print('paddlenlp commit:',paddlenlp.version.commit)" >> ${log_path}/commit_info.txt
 
@@ -127,7 +125,6 @@ if [[ ${#Build_list[*]} -ne 0 ]];then
     fi
 
     if [ -e "${upload_path}" ] && [ "$(ls -A "${upload_path}/")" ]; then
-        python -m pip install bce-python-sdk==0.8.74 --trusted-host pip.baidu-int.com --force-reinstall
         cd ${upload_path} && ls -A "${upload_path}"
         cd ${PPNLP_HOME} && python upload.py ${upload_path} 'paddlenlp/wheels'
         rm -rf ${upload_path}

tests/llm/test_grpo.py

@@ -92,7 +92,7 @@ def test_grpo(self):
             # 运行主逻辑
             cmd = 'python -u -m paddle.distributed.launch \
                     --devices "$CUDA_VISIBLE_DEVICES" run_rl.py \
-                    ../../config/qwen/reinforce_plus_plus_argument.yaml \
+                    ../../config/qwen/grpo_argument.yaml \
                     --actor_model_name_or_path "Qwen/Qwen2-1.5B" \
                     --max_dec_len 128 \
                     --max_steps 3 \

tests/llm/test_reinforce_plus_plus.py

@@ -92,7 +92,7 @@ def test_reinforce_plus_plus(self):
             # 运行主逻辑
             cmd = 'python -u -m paddle.distributed.launch \
                     --devices "$CUDA_VISIBLE_DEVICES" run_rl.py \
-                    ../../config/qwen/reinforce_plus_plus_argument.yaml \
+                    ../../config/qwen/grpo_argument.yaml \
                     --rl_algorithm "reinforce_plus_plus" \
                     --actor_model_name_or_path "Qwen/Qwen2-1.5B" \
                     --max_dec_len 128 \

tests/utils/test_aistudio_download.py

@@ -17,8 +17,11 @@
 from paddlenlp.transformers.aistudio_utils import aistudio_download
 from paddlenlp.utils.download import resolve_file_path
 
+from ..testing_utils import skip_for_none_ce_case
+
 
 class TestAistudioDownload(unittest.TestCase):
+    @skip_for_none_ce_case
     def test_aistudio_download(self):
         # 设置测试数据
         repo_id = "PaddleNLP/DeepSeek-R1-Distill-Qwen-1.5B"
@@ -39,6 +42,7 @@ def test_aistudio_download(self):
         print(result)
         self.assertEqual(result, f"{local_dir}/{filename}")
 
+    @skip_for_none_ce_case
     def test_aistudio_download_transformer(self):
         # 设置测试数据
         repo_id = "PaddleNLP/DeepSeek-R1-Distill-Qwen-1.5B"