Skip to content

Commit 4bb4677

Browse files
rainyflyrainyfly
committed
Merge branch 'develop' of https://github.com/PaddlePaddle/FastDeploy into set_v1_scheduler_as_default
2 parents 9c4a26a + 205b706 commit 4bb4677

File tree

148 files changed

+3989
-672
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

148 files changed

+3989
-672
lines changed

‎.github/workflows/_accuracy_test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ jobs:
4444
FULL_REPO="${{ github.repository }}"
4545
REPO_NAME="${FULL_REPO##*/}"
4646
BASE_BRANCH="${{ github.base_ref }}"
47-
47+
docker pull ${docker_image}
4848
# Clean the repository directory before starting
4949
docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
5050
-e "REPO_NAME=${REPO_NAME}" \

‎.github/workflows/_base_test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ jobs:
4444
FULL_REPO="${{ github.repository }}"
4545
REPO_NAME="${FULL_REPO##*/}"
4646
BASE_BRANCH="${{ github.base_ref }}"
47-
47+
docker pull ${docker_image}
4848
# Clean the repository directory before starting
4949
docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
5050
-e "REPO_NAME=${REPO_NAME}" \

‎.github/workflows/_ci_image_build.yml‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
name: Docker Build
2+
description: "FastDeploy CI Image Build"
3+
4+
on:
5+
workflow_call:
6+
inputs:
7+
CI_DOCKER_IMAGE_NAME:
8+
description: "Build Images"
9+
required: true
10+
type: string
11+
default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:cuda126-py310"
12+
FASTDEPLOY_ARCHIVE_URL:
13+
description: "URL of the compressed FastDeploy code archive."
14+
required: true
15+
type: string
16+
DOCKER_IMAGE_NAME:
17+
description: "Build Images"
18+
required: false
19+
type: string
20+
default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate"
21+
outputs:
22+
docker_name_precheck:
23+
description: "Output path of the generated wheel"
24+
value: ${{ jobs.docker_build.outputs.docker_name_precheck }}
25+
26+
jobs:
27+
docker_build:
28+
runs-on: [self-hosted, Docker-Build]
29+
outputs:
30+
docker_name_precheck: ${{ steps.docker_build.outputs.docker_name_precheck }}
31+
steps:
32+
- name: Code Prepare
33+
id: docker_build
34+
shell: bash
35+
env:
36+
docker_image_name: ${{ inputs.CI_DOCKER_IMAGE_NAME }}
37+
docker_image: ${{ inputs.DOCKER_IMAGE_NAME }}
38+
fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }}
39+
run: |
40+
set -x
41+
REPO="https://github.com/${{ github.repository }}.git"
42+
FULL_REPO="${{ github.repository }}"
43+
REPO_NAME="${FULL_REPO##*/}"
44+
BASE_BRANCH="${{ github.base_ref }}"
45+
46+
# Clean the repository directory before starting
47+
docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
48+
-e "REPO_NAME=${REPO_NAME}" \
49+
${docker_image} /bin/bash -c '
50+
if [ -d ${REPO_NAME} ]; then
51+
echo "Directory ${REPO_NAME} exists, removing it..."
52+
rm -rf ${REPO_NAME}*
53+
fi
54+
'
55+
56+
wget -q ${fd_archive_url}
57+
tar -xf FastDeploy.tar.gz
58+
rm -rf FastDeploy.tar.gz
59+
cd FastDeploy
60+
git config --global user.name "FastDeployCI"
61+
git config --global user.email "fastdeploy_ci@example.com"
62+
git log -n 3 --oneline
63+
64+
# Docker Build
65+
cd tools/dockerfile/
66+
set -e
67+
cp ../../requirements.txt ./
68+
cp ../../scripts/unittest_requirement.txt ./
69+
docker build -t ${docker_image_name} -f Dockerfile.ci . \
70+
--network host \
71+
--no-cache
72+
docker push ${docker_image_name}
73+
echo "docker_name_precheck=${docker_image_name}" >> $GITHUB_OUTPUT

‎.github/workflows/_logprob_test_linux.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ jobs:
3939
docker_image: ${{ inputs.DOCKER_IMAGE }}
4040
paddletest_archive_url: ${{ inputs.PADDLETEST_ARCHIVE_URL }}
4141
run: |
42+
docker pull ${docker_image}
4243
# Clean the repository directory before starting
4344
docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
4445
-e "REPO_NAME=${REPO_NAME}" \
@@ -114,7 +115,6 @@ jobs:
114115
echo "Removing stale container: ${runner_name}"
115116
docker rm -f ${runner_name} || true
116117
fi
117-
118118
docker run --rm --ipc=host --pid=host --net=host \
119119
--name ${runner_name} \
120120
-v $(pwd):/workspace \

‎.github/workflows/_pre_ce_test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ jobs:
4646
FULL_REPO="${{ github.repository }}"
4747
REPO_NAME="${FULL_REPO##*/}"
4848
BASE_BRANCH="${{ github.base_ref }}"
49-
49+
docker pull ${docker_image}
5050
# Clean the repository directory before starting
5151
docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
5252
-e "REPO_NAME=${REPO_NAME}" \

‎.github/workflows/_stable_test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ jobs:
4444
FULL_REPO="${{ github.repository }}"
4545
REPO_NAME="${FULL_REPO##*/}"
4646
BASE_BRANCH="${{ github.base_ref }}"
47-
47+
docker pull ${docker_image}
4848
# Clean the repository directory before starting
4949
docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
5050
-e "REPO_NAME=${REPO_NAME}" \

‎.github/workflows/_unit_test_coverage.yml‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ jobs:
6060
FULL_REPO="${{ github.repository }}"
6161
REPO_NAME="${FULL_REPO##*/}"
6262
BASE_BRANCH="${{ github.base_ref }}"
63-
63+
docker pull ${docker_image}
6464
# Clean the repository directory before starting
6565
docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
6666
-e "REPO_NAME=${REPO_NAME}" \
@@ -164,15 +164,19 @@ jobs:
164164
165165
git config --global --add safe.directory /workspace/FastDeploy
166166
cd FastDeploy
167+
git diff origin/${BASE_REF}..HEAD --unified=0 > diff.txt
167168
python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
168-
169169
pip config set global.extra-index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
170170
171171
python -m pip install coverage
172172
python -m pip install diff-cover
173173
python -m pip install pytest-cov
174174
python -m pip install jsonschema aistudio_sdk==0.3.5
175175
python -m pip install ${fd_wheel_url}
176+
rm -rf fastdeploy
177+
# coverage subprocess use
178+
python -m pip install ${fd_wheel_url} --no-deps --target=/workspace/FastDeploy
179+
export PYTHONPATH=/workspace/FastDeploy/
176180
if [ -d "tests/plugins" ]; then
177181
cd tests/plugins
178182
python setup.py install
@@ -182,12 +186,11 @@ jobs:
182186
fi
183187
export COVERAGE_FILE=/workspace/FastDeploy/coveragedata/.coverage
184188
export COVERAGE_RCFILE=/workspace/FastDeploy/scripts/.coveragerc
185-
export COVERAGE_PROCESS_START=/workspace/FastDeploy/scripts/.coveragerc
186189
TEST_EXIT_CODE=0
187190
bash scripts/coverage_run.sh || TEST_EXIT_CODE=8
188-
git diff origin/${BASE_REF}..HEAD --unified=0 > diff.txt
189191
echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> exit_code.env
190192
coverage combine coveragedata/ || echo "No data to combine"
193+
coverage report
191194
coverage xml -o python_coverage_all.xml
192195
COVERAGE_EXIT_CODE=0
193196
if [[ "$IS_PR" == "true" ]]; then
@@ -228,7 +231,7 @@ jobs:
228231
echo "diff_cov_result_json_url=${DIFF_COV_JSON_URL}" >> $GITHUB_OUTPUT
229232
echo "diff_cov_result_json_url=${DIFF_COV_JSON_URL}" >> $GITHUB_ENV
230233
fi
231-
unittest_result="tests/failed_tests.log"
234+
unittest_result="failed_tests.log"
232235
if [ -s ${unittest_result} ];then
233236
python ${push_file} ${unittest_result} ${target_path}/UnitTestResult
234237
target_path_stripped="${target_path#paddle-github-action/}"

‎.github/workflows/ci_image_update.yml‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
name: CI Images Build
2+
3+
on:
4+
workflow_dispatch:
5+
schedule:
6+
- cron: '0 18 * * *' # 2:00 AM China Standard Time (UTC+8)
7+
8+
permissions: read-all
9+
10+
concurrency:
11+
group: ${{ github.ref }}-${{ github.sha }}
12+
cancel-in-progress: true
13+
14+
15+
jobs:
16+
clone:
17+
environment: CodeSync
18+
name: FD-Clone-Linux
19+
runs-on: ubuntu-latest
20+
outputs:
21+
repo_archive_url: ${{ steps.set_output.outputs.repo_archive_url }}
22+
steps:
23+
- name: Clone FastDeploy
24+
uses: actions/checkout@v4
25+
with:
26+
ref: ${{ github.ref_name }}
27+
submodules: 'recursive'
28+
fetch-depth: 1000
29+
30+
- name: Python Setup
31+
uses: actions/setup-python@v5
32+
with:
33+
python-version: '3.10'
34+
- name: Code Info Show and Upload
35+
id: set_output
36+
env:
37+
AK: ${{ secrets.BOS_AK }}
38+
SK: ${{ secrets.BOS_SK }}
39+
run: |
40+
git config --unset http.https://github.com/.extraheader
41+
git submodule foreach --recursive sh -c "git config --local --unset-all 'http.https://github.com/.extraheader'"
42+
git submodule foreach --recursive sh -c "git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'"
43+
echo "Current HEAD Log:"
44+
git log --oneline -n 5
45+
ls
46+
cd ..
47+
tar -zcf FastDeploy.tar.gz FastDeploy
48+
if [[ "${{ github.ref_type }}" == "tag" ]]; then
49+
commit_id=${{ github.sha }}
50+
tag_name=${{ github.ref_name }}
51+
target_path=paddle-qa/TAG/FastDeploy/${tag_name}/${commit_id}
52+
else
53+
commit_id=${{ github.sha }}
54+
branch_name=${{ github.ref_name }}
55+
target_path=paddle-qa/BRANCH/FastDeploy/${branch_name}/${commit_id}
56+
fi
57+
wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
58+
push_file=$(realpath bos_tools.py)
59+
python -m pip install bce-python-sdk==0.9.29
60+
ls
61+
python ${push_file} FastDeploy.tar.gz ${target_path}
62+
target_path_stripped="${target_path#paddle-qa/}"
63+
REPO_ARCHIVE_URL=https://paddle-qa.bj.bcebos.com/${target_path_stripped}/FastDeploy.tar.gz
64+
echo "repo_archive_url=${REPO_ARCHIVE_URL}" >> $GITHUB_OUTPUT
65+
66+
resultshow:
67+
name: Show Code Archive Output
68+
needs: clone
69+
runs-on: ubuntu-latest
70+
steps:
71+
- name: Print wheel path
72+
run: |
73+
echo "The code archive is located at: ${{ needs.clone.outputs.repo_archive_url }}"
74+
75+
ci_image_build:
76+
name: CI Images Build
77+
needs: clone
78+
uses: ./.github/workflows/_ci_image_build.yml
79+
with:
80+
CI_DOCKER_IMAGE_NAME: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate-precheck
81+
FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
82+
83+
84+
build_sm8090:
85+
name: BUILD_SM8090
86+
needs: [clone, ci_image_build]
87+
uses: ./.github/workflows/_build_linux.yml
88+
with:
89+
DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }}
90+
FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
91+
COMPILE_ARCH: "90"
92+
WITH_NIGHTLY_BUILD: ${{ needs.publish_pre_check.outputs.with_nightly_build }}
93+
FD_VERSION: ${{ needs.publish_pre_check.outputs.fd_version }}
94+
PADDLEVERSION: ${{ needs.publish_pre_check.outputs.compile_use_paddle_version }}
95+
PADDLE_WHL_URL: ${{ needs.publish_pre_check.outputs.compile_use_paddle_whl_url }}
96+
97+
98+
unittest_coverage:
99+
name: Run FastDeploy Unit Tests and Coverage
100+
needs: [clone,build_sm8090,ci_image_build]
101+
uses: ./.github/workflows/_unit_test_coverage.yml
102+
with:
103+
DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }}
104+
FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
105+
FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
106+
MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"
107+
secrets:
108+
github-token: ${{ secrets.GITHUB_TOKEN }}
109+
110+
logprob_test:
111+
name: Run FastDeploy LogProb Tests
112+
needs: [build_sm8090,ci_image_build]
113+
uses: ./.github/workflows/_logprob_test_linux.yml
114+
with:
115+
DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }}
116+
PADDLETEST_ARCHIVE_URL: "https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz"
117+
FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
118+
MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"
119+
120+
pre_ce_test:
121+
name: Extracted partial CE model tasks to run in CI.
122+
needs: [clone,build_sm8090,ci_image_build]
123+
uses: ./.github/workflows/_pre_ce_test.yml
124+
with:
125+
DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }}
126+
FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
127+
FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
128+
MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"
129+
130+
base_test:
131+
name: Run Base Tests
132+
needs: [clone,build_sm8090,ci_image_build]
133+
uses: ./.github/workflows/_base_test.yml
134+
with:
135+
DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }}
136+
FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
137+
FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
138+
MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"
139+
140+
accuracy_test:
141+
name: Run Accuracy Tests
142+
needs: [clone,build_sm8090,ci_image_build]
143+
uses: ./.github/workflows/_accuracy_test.yml
144+
with:
145+
DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }}
146+
FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
147+
FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
148+
MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"
149+
150+
stable_test:
151+
name: Run Stable Tests
152+
needs: [clone,build_sm8090,ci_image_build]
153+
uses: ./.github/workflows/_stable_test.yml
154+
with:
155+
DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }}
156+
FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
157+
FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
158+
MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"
159+
160+
161+
publish_pre_check:
162+
name: Publish Docker Images Pre Check
163+
needs: [ci_image_build, unittest_coverage,logprob_test,pre_ce_test,base_test,accuracy_test,stable_test]
164+
runs-on: [self-hosted, Docker-Build]
165+
steps:
166+
- name: Images Uploading
167+
env:
168+
images_name: ${{ needs.ci_image_build.outputs.docker_name_precheck }}
169+
ci_image_name: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate"
170+
run: |
171+
echo "images_name=${images_name}"
172+
docker images ${ci_image_name}
173+
docker tag ${images_name} ${ci_image_name}
174+
docker push ${ci_image_name}

‎.github/workflows/pr_build_and_test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ jobs:
2121
with:
2222
DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
2323
FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
24-
COMPILE_ARCH: "89,90"
24+
COMPILE_ARCH: "90"
2525
WITH_NIGHTLY_BUILD: "OFF"
2626
FD_VERSION: "0.0.0"
2727

‎.github/workflows/publish_job.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,3 +319,13 @@ jobs:
319319
FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
320320
FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
321321
MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"
322+
323+
stable_test:
324+
name: Run Stable Tests
325+
needs: [clone,build]
326+
uses: ./.github/workflows/_stable_test.yml
327+
with:
328+
DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
329+
FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
330+
FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }}
331+
MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"

0 commit comments

Comments
 (0)