Skip to content

Commit b54902c

Browse files
Liujie0926Liujie0926
authored
[CI] Update image (#2542)
1 parent c5c830b commit b54902c

File tree

7 files changed

+40
-57
lines changed

7 files changed

+40
-57
lines changed

‎.github/workflows/build-ci-docker-images.yml‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ on:
1212
# types: [opened, synchronize, reopened]
1313
# branches: [develop]
1414
schedule:
15-
- cron: "5 0 * * 0" # every Sunday at 00:05
15+
- cron: "0 16 * * 5" # UTC 16:00 = Beijing Time, Saturday at 00:00
1616
workflow_dispatch: # allow to manually trigger the workflow
1717

1818
concurrency:
@@ -25,14 +25,13 @@ env:
2525
BRANCH: ${{ github.event.pull_request.base.ref }}
2626
GITHUB_EVENT_NAME: ${{ github.event_name }}
2727
TASK: PaddleFormers-CI-${{ github.event.pull_request.number }}-build-image
28-
image_base: "iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu18.04-gcc8.2-cuda11.8-cudnn8.6-nccl2.15.5"
29-
no_proxy: "localhost,bj.bcebos.com,su.bcebos.com,bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
28+
image_base: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda126-dev"
3029

3130
jobs:
3231
build-ci-docker-images:
3332
name: build-ci-docker-images
3433
if: github.ref == 'refs/heads/develop'
35-
runs-on: [self-hosted, ernie-8gpu]
34+
runs-on: [self-hosted, ernie-8gpu-1]
3635
outputs:
3736
run_build: ${{ steps.set_flags.outputs.run_build }}
3837
run_downstream: ${{ steps.set_flags.outputs.run_downstream }}
@@ -45,7 +44,7 @@ jobs:
4544
run: |
4645
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
4746
echo "container_name=${container_name}" >> ${{ github.env }}
48-
docker_image="${image_base}-latest"
47+
docker_image="${image_base}"
4948
docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
5049
-v $work_dir/../../..:$work_dir/../../.. \
5150
-v $work_dir:/workspace \
@@ -55,7 +54,7 @@ jobs:
5554
-e COMMIT_ID \
5655
-e work_dir \
5756
-e no_proxy \
58-
-w /workspace --runtime=nvidia ${docker_image}
57+
-w /workspace --runtime=nvidia --privileged ${docker_image}
5958
6059
- name: Download Code
6160
env:
@@ -139,10 +138,9 @@ jobs:
139138
- name: Write Dockerfile Inline
140139
run: |
141140
cat <<EOF > Dockerfile
142-
FROM iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu18.04-gcc8.2-cuda11.8-cudnn8.6-nccl2.15.5-libsndfile
143-
RUN unlink /usr/bin/python3 && ln -sf python3.10 /usr/bin/python3
144-
RUN rm -rf /etc/pip.conf
141+
FROM ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda126-dev
145142
RUN python -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
143+
RUN python -m pip config set global.trusted-host pypi.tuna.tsinghua.edu.cn
146144
RUN python -m pip install bce-python-sdk==0.8.74
147145
COPY PaddleFormers/ /tmp/paddleformers/
148146
WORKDIR /tmp/paddleformers
@@ -153,7 +151,6 @@ jobs:
153151
RUN python -m pip install dist/*.whl
154152
WORKDIR /
155153
RUN rm -rf /tmp/paddleformers
156-
CMD ["python3"]
157154
EOF
158155
159156
- name: Build Docker Image
@@ -180,7 +177,7 @@ jobs:
180177
clean-ci-image:
181178
name: clean-ci-image
182179
needs: [build-ci-docker-images, test-ci-docker-images]
183-
runs-on: [self-hosted, ernie-8gpu]
180+
runs-on: [self-hosted, ernie-8gpu-1]
184181
steps:
185182
- name: Remove CI Image
186183
env:
@@ -203,5 +200,5 @@ jobs:
203200
echo "$CONTAINERS"
204201
fi
205202
fi
206-
# 安全删除悬空镜像（无悬空镜像时跳过）
203+
# Safely remove dangling images (skip if no dangling images exist)
207204
docker images -f "dangling=true" -q | xargs -r docker rmi -f

‎.github/workflows/ce-build.yml‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,14 @@ jobs:
2222
steps:
2323
- name: Determine Image Name
2424
run: |
25-
echo "IMAGE_NAME=iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu18.04-gcc8.2-cuda11.8-cudnn8.6-nccl2.15.5-latest" >> "$GITHUB_ENV"
25+
echo "IMAGE_NAME=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda126-dev-latest" >> "$GITHUB_ENV"
2626
2727
- name: Run Container
2828
env:
2929
work_dir: ${{ github.workspace }}
3030
FLAGS_dynamic_static_unified_comm: "True"
3131
python_version: "3.10"
32-
paddle: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CINN-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
32+
paddle: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuAll-LinuxCentos-Gcc11-Cuda126-Cudnn95-Trt105-Py310-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
3333
run: |
3434
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
3535
echo "container_name=${container_name}" >> "$GITHUB_ENV"
@@ -46,7 +46,7 @@ jobs:
4646
-e paddle \
4747
-e FLAGS_dynamic_static_unified_comm \
4848
-e python_version \
49-
-w /workspace --runtime=nvidia $IMAGE_NAME
49+
-w /workspace --runtime=nvidia --privileged $IMAGE_NAME
5050
5151
- name: Download Code
5252
run: |
@@ -70,8 +70,6 @@ jobs:
7070
run: |
7171
docker exec -t $container_name /bin/bash -c '
7272
ldconfig
73-
unlink /usr/bin/python3
74-
ln -sf $(which python${python_version}) /usr/bin/python3
7573
pip config set global.cache-dir "/home/.cache/pip"
7674
set -e
7775
cd /workspace/PaddleFormers && git config --global --add safe.directory $PWD

‎.github/workflows/ce-deadlink.yml‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ name: Deadlink CE
22

33
on:
44
schedule:
5-
- cron: "0 8 * * 6" # every Saturday at 6:00 AM
5+
- cron: "0 8 * * 6" # every Saturday at 16:00
66
workflow_dispatch: # allow to manually trigger the workflow
77

88
concurrency:
@@ -28,14 +28,14 @@ jobs:
2828
steps:
2929
- name: Determine Image Name
3030
run: |
31-
echo "IMAGE_NAME=iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu18.04-gcc8.2-cuda11.8-cudnn8.6-nccl2.15.5-latest" >> "$GITHUB_ENV"
31+
echo "IMAGE_NAME=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda126-dev-latest" >> "$GITHUB_ENV"
3232
3333
- name: Run Container
3434
env:
3535
work_dir: ${{ github.workspace }}
3636
FLAGS_dynamic_static_unified_comm: "True"
3737
python_version: "3.10"
38-
paddle_whl: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CENN-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
38+
paddle_whl: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuAll-LinuxCentos-Gcc11-Cuda126-Cudnn95-Trt105-Py310-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
3939
run: |
4040
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
4141
echo "container_name=${container_name}" >> "$GITHUB_ENV"
@@ -52,7 +52,7 @@ jobs:
5252
-e paddle_whl \
5353
-e FLAGS_dynamic_static_unified_comm \
5454
-e python_version \
55-
-w /workspace --runtime=nvidia $IMAGE_NAME
55+
-w /workspace --runtime=nvidia --privileged $IMAGE_NAME
5656
5757
- name: Download Code
5858
run: |
@@ -80,13 +80,11 @@ jobs:
8080
run: |
8181
docker exec -t $container_name /bin/bash -c '
8282
ldconfig
83-
unlink /usr/bin/python3
84-
ln -sf $(which python${python_version}) /usr/bin/python3
8583
pip config set global.cache-dir "/home/.cache/pip"
8684
set -e
8785
python -m pip install beautifulsoup4 openpyxl
8886
source $work_dir/../../../proxy
89-
bash run.sh PaddleFormers develop [email protected] PaddleFormers死链检测汇总报告
87+
bash run.sh PaddleFormers develop [email protected] "PaddleFormers Broken Link Check Summary Report"
9088
'
9189
9290
- name: Upload Logs
@@ -102,8 +100,8 @@ jobs:
102100
tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
103101
fi
104102
bos_prefix="schedule/$(date +%Y%m%d)"
105-
cd /workspace/PaddleFormers/result
106-
for FILE in /workspace/PaddleFormers/result/*; do
103+
cd /workspace/result
104+
for FILE in /workspace/result/*; do
107105
file=$(basename "$FILE")
108106
python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/deadlink/${bos_prefix}/logs
109107
echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/deadlink/${bos_prefix}/logs/$file"

‎.github/workflows/ce-unittest-gpu.yml‎

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,16 @@ name: Unittest GPU CE
22

33
on:
44
schedule:
5-
- cron: "0 6 * * 6"
5+
- cron: "0 2 * * 6" # every Saturday at 10:00
66
workflow_dispatch: # allow to manually trigger the workflow
77
inputs:
88
paddle_whl:
9-
description: "paddle链接"
9+
description: "paddle_whl"
1010
required: false
11-
default: "https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CINN-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl"
11+
default: "https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuAll-LinuxCentos-Gcc11-Cuda126-Cudnn95-Trt105-Py310-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl"
1212
type: string
1313
repo_branch:
14-
description: "套件分支"
14+
description: "repo_branch"
1515
required: false
1616
type: string
1717

@@ -35,19 +35,18 @@ defaults:
3535
jobs:
3636
unittest-gpu-ce:
3737
name: unittest-gpu-ce
38-
if: github.ref == 'refs/heads/develop'
3938
runs-on: [self-hosted, ernie-8gpu]
4039
steps:
4140
- name: Determine Image Name
4241
run: |
43-
echo "IMAGE_NAME=iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu18.04-gcc8.2-cuda11.8-cudnn8.6-nccl2.15.5-latest" >> "$GITHUB_ENV"
42+
echo "IMAGE_NAME=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda126-dev-latest" >> "$GITHUB_ENV"
4443
4544
- name: Run Container
4645
env:
4746
work_dir: ${{ github.workspace }}
4847
FLAGS_dynamic_static_unified_comm: "True"
4948
python_version: "3.10"
50-
paddle_whl: ${{ github.event.inputs.paddle_whl || 'https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CINN-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl' }}
49+
paddle_whl: ${{ github.event.inputs.paddle_whl || 'https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuAll-LinuxCentos-Gcc11-Cuda126-Cudnn95-Trt105-Py310-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl' }}
5150
repo_branch: ${{ github.event.inputs.repo_branch || '' }}
5251
run: |
5352
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
@@ -67,7 +66,7 @@ jobs:
6766
-e repo_branch \
6867
-e FLAGS_dynamic_static_unified_comm \
6968
-e python_version \
70-
-w /workspace --runtime=nvidia $IMAGE_NAME
69+
-w /workspace --runtime=nvidia --privileged $IMAGE_NAME
7170
7271
- name: Download Code
7372
run: |
@@ -98,12 +97,10 @@ jobs:
9897
run: |
9998
docker exec -t $container_name /bin/bash -c '
10099
ldconfig
101-
unlink /usr/bin/python3
102-
ln -sf $(which python${python_version}) /usr/bin/python3
103100
pip config set global.cache-dir "/home/.cache/pip"
104101
set -e
105102
cd /workspace/PaddleFormers && git config --global --add safe.directory $PWD
106-
timeout 2h bash scripts/unit_test/ci_unit.sh ${paddle} true
103+
timeout 2h bash scripts/unit_test/ci_unit.sh ${paddle_whl} true
107104
'
108105
109106
- name: Upload Allure-reports & Logs

‎.github/workflows/debug-unittest-gpu.yml‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,11 @@ jobs:
2828
work_dir: ${{ github.workspace }}
2929
FLAGS_dynamic_static_unified_comm: "True"
3030
python_version: "3.10"
31-
paddle_whl: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CINN-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
31+
paddle_whl: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuAll-LinuxCentos-Gcc11-Cuda126-Cudnn95-Trt105-Py310-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
3232
run: |
3333
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
3434
echo "container_name=${container_name}" >> "$GITHUB_ENV"
35-
image_name=iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu18.04-gcc8.2-cuda11.8-cudnn8.6-nccl2.15.5-latest
35+
image_name=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda126-dev-latest
3636
echo "image_name=${image_name}" >> "$GITHUB_ENV"
3737
docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
3838
-v $work_dir/../../..:$work_dir/../../.. \
@@ -48,7 +48,7 @@ jobs:
4848
-e paddle_whl \
4949
-e FLAGS_dynamic_static_unified_comm \
5050
-e python_version \
51-
-w /workspace --runtime=nvidia $image_name
51+
-w /workspace --runtime=nvidia --privileged $image_name
5252
5353
- name: Download Code
5454
run: |
@@ -83,8 +83,6 @@ jobs:
8383
docker exec -t $container_name /bin/bash -c '
8484
ldconfig
8585
set -e
86-
unlink /usr/bin/python3
87-
ln -sf $(which python${python_version}) /usr/bin/python3
8886
python -c "import sys; print(sys.version_info[:])"
8987
pip config set global.cache-dir "/home/.cache/pip"
9088
cd /workspace/PaddleFormers && git config --global --add safe.directory $PWD
@@ -94,13 +92,10 @@ jobs:
9492
env:
9593
work_dir: ${{ github.workspace }}
9694
run: |
97-
echo "容器已启动，请登录机器进行调试"
98-
echo "登录机器后，进入容器："
9995
echo "docker exec -it $container_name bash"
100-
echo "物理机本地工作目录为：$work_dir/PaddleFormers"
101-
echo "容器内工作目录为：/workspace/PaddleFormers"
102-
echo "unittest-gpu任务整体复现命令"
96+
echo "work_path: $work_dir/PaddleFormers"
97+
echo "work_path in docker: /workspace/PaddleFormers"
98+
echo "cmd: "
10399
echo "bash scripts/unit_test/ci_unit.sh ${paddle_whl}"
104-
echo "也可直接在$work_dir下执行python -m pytest 失败case"
105-
echo "请调试完成后手动执行清理命令"
100+
echo "or python -m pytest fail_case_name"
106101
echo "docker rm -f $container_name"

‎.github/workflows/unittest-gpu.yml‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,15 +44,15 @@ jobs:
4444
if [[ -n "${IMAGE_NAME}" ]]; then
4545
echo "IMAGE_NAME=${IMAGE_NAME}" >> "$GITHUB_ENV"
4646
else
47-
echo "IMAGE_NAME=iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu18.04-gcc8.2-cuda11.8-cudnn8.6-nccl2.15.5-latest" >> "$GITHUB_ENV"
47+
echo "IMAGE_NAME=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda126-dev-latest" >> "$GITHUB_ENV"
4848
fi
4949
5050
- name: Run Container
5151
env:
5252
work_dir: ${{ github.workspace }}
5353
FLAGS_dynamic_static_unified_comm: "True"
5454
python_version: "3.10"
55-
paddle_whl: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CINN-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
55+
paddle_whl: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuAll-LinuxCentos-Gcc11-Cuda126-Cudnn95-Trt105-Py310-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
5656
run: |
5757
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
5858
echo "container_name=${container_name}" >> "$GITHUB_ENV"
@@ -76,7 +76,7 @@ jobs:
7676
-e python_version \
7777
-e HF_PROXY_PATH=$work_dir/../../../proxy_huggingface \
7878
-e AISTUDIO_PROXY_PATH=$work_dir/../../../proxy_aistudio \
79-
-w /workspace --runtime=nvidia $IMAGE_NAME
79+
-w /workspace --runtime=nvidia --privileged $IMAGE_NAME
8080
fi
8181
8282
- name: Download Code
@@ -117,8 +117,6 @@ jobs:
117117
else
118118
docker exec -t $container_name /bin/bash -c '
119119
ldconfig
120-
unlink /usr/bin/python3
121-
ln -sf $(which python${python_version}) /usr/bin/python3
122120
pip config set global.cache-dir "/home/.cache/pip"
123121
set -e
124122
rm -rf /root/.cache/aistudio/

‎scripts/unit_test/ci_unit.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ install_requirements() {
3232
python -m pip install -r requirements-dev.txt
3333
python -m pip install -r tests/requirements.txt
3434
python -m pip uninstall paddlepaddle paddlepaddle_gpu -y
35-
python -m pip install --no-cache-dir ${paddle}
35+
python -m pip install --no-cache-dir ${paddle} --no-dependencies
3636
python -c "import paddle;print('paddle');print(paddle.__version__);print(paddle.version.show())" >> ${log_path}/commit_info.txt
3737

3838
python setup.py bdist_wheel > /dev/null
@@ -82,14 +82,14 @@ print_info() {
8282
get_diff_TO_case(){
8383
export FLAGS_enable_CI=false
8484
if [ -z "${AGILE_COMPILE_BRANCH}" ]; then
85-
# 定时任务回归测试
85+
# Scheduled Regression Test
8686
FLAGS_enable_CI=true
8787
else
8888
for file_name in `git diff --numstat ${AGILE_COMPILE_BRANCH} -- |awk '{print $NF}'`;do
8989
ext="${file_name##*.}"
9090
echo "file_name: ${file_name}, ext: ${file_name##*.}"
9191

92-
if [ ! -f ${file_name} ];then # 针对pr删掉文件
92+
if [ ! -f ${file_name} ];then # Delete Files for a Pull Request
9393
continue
9494
elif [[ "$ext" == "md" || "$ext" == "rst" || "$file_name" == docs/* ]]; then
9595
continue

0 commit comments

Comments
 (0)