Skip to content

Commit e113319

Browse files
Jiang-Jia-JunXieYunshenming1753zoooo0820EmmonsCurse
authored
[Sync Code] Update vs branch (#3403)
* Pre ce modified (#3335) (#3360) * Pre ce modified (#3335) * update * update * fix * fix * update * update * update * fix * update * update * update * add ut fix pr(3367) * [Bug Fix] Fix V1 video bug (#3387) * fix stopseq error info (#3342) Co-authored-by: YuBaoku <[email protected]> * [BugFix] Fix default log level of paddleformers (#3377) Co-authored-by: YuBaoku <[email protected]> * [Polish Code] Remove useless notes * feat(log):add_request_and_response_log (#3392) * Optimize CI execution workflow. (#3371) (#3384) * fix * [BugFix] fix control signal release failed (#3374) * [BugFix] * [BugFix] * [BugFix] * [BugFix] * fix * fix --------- Co-authored-by: YuBaoku <[email protected]> Co-authored-by: Jiang-Jia-Jun <[email protected]> --------- Co-authored-by: YUNSHEN XIE <[email protected]> Co-authored-by: ming1753 <[email protected]> Co-authored-by: JYChen <[email protected]> Co-authored-by: YuBaoku <[email protected]> Co-authored-by: Jiang-Jia-Jun <[email protected]> Co-authored-by: xiaolei373 <[email protected]> Co-authored-by: ltd0924 <[email protected]>
1 parent 81092c0 commit e113319

File tree

16 files changed

+334
-160
lines changed

16 files changed

+334
-160
lines changed

‎.github/workflows/_logprob_test_linux.yml

Lines changed: 33 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -62,18 +62,22 @@ jobs:
6262
MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }}
6363
run: |
6464
runner_name="${{ runner.name }}"
65-
last_char="${runner_name: -1}"
65+
CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
66+
DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
67+
DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1)
6668
67-
if [[ "$last_char" =~ [0-7] ]]; then
68-
DEVICES="$last_char"
69-
else
70-
DEVICES="0"
71-
fi
72-
73-
FLASK_PORT=$((9160 + DEVICES * 100))
74-
FD_API_PORT=$((9180 + DEVICES * 100))
75-
FD_ENGINE_QUEUE_PORT=$((9150 + DEVICES * 100))
76-
FD_METRICS_PORT=$((9170 + DEVICES * 100))
69+
FLASK_PORT=$((42068 + DEVICE_PORT * 100))
70+
FD_API_PORT=$((42088 + DEVICE_PORT * 100))
71+
FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
72+
FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
73+
echo "Test ENV Parameter:"
74+
echo "========================================================="
75+
echo "FLASK_PORT=${FLASK_PORT}"
76+
echo "FD_API_PORT=${FD_API_PORT}"
77+
echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
78+
echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
79+
echo "DEVICES=${DEVICES}"
80+
echo "========================================================="
7781
7882
CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
7983
echo "CACHE_DIR is set to ${CACHE_DIR}"
@@ -85,7 +89,24 @@ jobs:
8589
exit 1
8690
fi
8791
88-
PARENT_DIR=$(dirname "$WORKSPACE")
92+
PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
93+
LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
94+
echo "==== LOG_FILE is ${LOG_FILE} ===="
95+
96+
echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE
97+
98+
for port in "${PORTS[@]}"; do
99+
PIDS=$(lsof -t -i :$port || true)
100+
if [ -n "$PIDS" ]; then
101+
echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
102+
echo "$PIDS" | xargs -r kill -9
103+
echo "Port $port cleared" | tee -a $LOG_FILE
104+
else
105+
echo "Port $port is free" | tee -a $LOG_FILE
106+
fi
107+
done
108+
109+
echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE
89110
90111
docker run --ipc=host --pid=host --net=host \
91112
-v $(pwd):/workspace \

‎.github/workflows/_pre_ce_test.yml

Lines changed: 53 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,19 @@ on:
2121
required: false
2222
type: string
2323
default: ""
24+
MODEL_CACHE_DIR:
25+
description: "Cache Dir Use"
26+
required: false
27+
type: string
28+
default: ""
2429

2530
concurrency:
2631
group: ${{ github.event.pull_request.number }}
2732
cancel-in-progress: true
2833

2934
jobs:
3035
run_ce_cases:
31-
runs-on: [self-hosted, GPU-L20-4Card]
36+
runs-on: [self-hosted, PRE_CE_RUN_2Card]
3237
steps:
3338
- name: Print current runner name
3439
run: |
@@ -67,34 +72,64 @@ jobs:
6772
env:
6873
docker_image: ${{ inputs.DOCKER_IMAGE }}
6974
fd_wheel_url: ${{ inputs.FASTDEPLOY_WHEEL_URL }}
75+
CACHE_DIR: ${{ inputs.CACHE_DIR }}
76+
MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }}
7077
run: |
7178
runner_name="${{ runner.name }}"
72-
last_char="${runner_name: -1}"
79+
CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
80+
DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
81+
DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1)
82+
83+
FLASK_PORT=$((42068 + DEVICE_PORT * 100))
84+
FD_API_PORT=$((42088 + DEVICE_PORT * 100))
85+
FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
86+
FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
87+
echo "Test ENV Parameter:"
88+
echo "========================================================="
89+
echo "FLASK_PORT=${FLASK_PORT}"
90+
echo "FD_API_PORT=${FD_API_PORT}"
91+
echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
92+
echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
93+
echo "DEVICES=${DEVICES}"
94+
echo "========================================================="
7395
74-
if [ "${last_char}" = "1" ]; then
75-
gpu_id=2
76-
DEVICES="2,3"
77-
else
78-
gpu_id=0
79-
DEVICES="0,1"
96+
CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
97+
echo "CACHE_DIR is set to ${CACHE_DIR}"
98+
if [ ! -f "${CACHE_DIR}/gitconfig" ]; then
99+
touch "${CACHE_DIR}/gitconfig"
80100
fi
81-
FD_API_PORT=$((9180 + gpu_id * 100))
82-
FD_ENGINE_QUEUE_PORT=$((9150 + gpu_id * 100))
83-
FD_METRICS_PORT=$((9170 + gpu_id * 100))
84101
85-
PARENT_DIR=$(dirname "$WORKSPACE")
86-
echo "PARENT_DIR:$PARENT_DIR"
102+
PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
103+
LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
104+
echo "==== LOG_FILE is ${LOG_FILE} ===="
105+
106+
echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE
107+
108+
for port in "${PORTS[@]}"; do
109+
PIDS=$(lsof -t -i :$port || true)
110+
if [ -n "$PIDS" ]; then
111+
echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
112+
echo "$PIDS" | xargs -r kill -9
113+
echo "Port $port cleared" | tee -a $LOG_FILE
114+
else
115+
echo "Port $port is free" | tee -a $LOG_FILE
116+
fi
117+
done
118+
119+
echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE
120+
87121
docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
88-
-v "/ssd4/GithubActions/gitconfig:/etc/gitconfig:ro" \
89-
-v "/ssd4/GithubActions/ModelData:/ModelData:ro" \
90-
-v "/ssd4/GithubActions/CacheDir:/root/.cache" \
91-
-v "/ssd4/GithubActions/ConfigDir:/root/.config" \
122+
-v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
123+
-v "${CACHE_DIR}/.cache:/root/.cache" \
124+
-v "${CACHE_DIR}/ConfigDir:/root/.config" \
125+
-v "${MODEL_CACHE_DIR}:/ModelData:ro" \
92126
-e "MODEL_PATH=/ModelData" \
93127
-e "FD_API_PORT=${FD_API_PORT}" \
94128
-e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
95129
-e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
130+
-e "FLASK_PORT=${FLASK_PORT}" \
96131
-e "fd_wheel_url=${fd_wheel_url}" \
97-
--gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -c '
132+
--gpus "\"device=${DEVICES}\"" ${docker_image} /bin/bash -c '
98133
git config --global --add safe.directory /workspace/FastDeploy
99134
cd FastDeploy
100135
python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/

‎.github/workflows/_unit_test_coverage.yml

Lines changed: 114 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@ on:
2222
required: false
2323
type: string
2424
default: ""
25+
MODEL_CACHE_DIR:
26+
description: "Cache Dir Use"
27+
required: false
28+
type: string
29+
default: ""
2530

2631
jobs:
2732
run_tests_with_coverage:
@@ -67,56 +72,102 @@ jobs:
6772
fd_wheel_url: ${{ inputs.FASTDEPLOY_WHEEL_URL }}
6873
CACHE_DIR: ${{ inputs.CACHE_DIR }}
6974
BASE_REF: ${{ github.event.pull_request.base.ref }}
75+
MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }}
7076
run: |
71-
set -x
72-
runner_name="${{ runner.name }}"
73-
CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
74-
gpu_id=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
75-
76-
CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
77-
echo "CACHE_DIR is set to ${CACHE_DIR}"
78-
if [ ! -f "${CACHE_DIR}/gitconfig" ]; then
79-
touch "${CACHE_DIR}/gitconfig"
80-
fi
81-
PARENT_DIR=$(dirname "$WORKSPACE")
82-
echo "PARENT_DIR:$PARENT_DIR"
83-
docker run --rm --net=host \
84-
--cap-add=SYS_PTRACE --privileged --shm-size=64G \
85-
-v $(pwd):/workspace -w /workspace \
86-
-v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
87-
-v "${CACHE_DIR}/.cache:/root/.cache" \
88-
-v "${CACHE_DIR}/ConfigDir:/root/.config" \
89-
-e TZ="Asia/Shanghai" \
90-
-e "fd_wheel_url=${fd_wheel_url}" \
91-
-e "BASE_REF=${BASE_REF}" \
92-
--gpus "\"device=${gpu_id}\"" ${docker_image} /bin/bash -c '
93-
94-
git config --global --add safe.directory /workspace/FastDeploy
95-
cd FastDeploy
96-
python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
97-
98-
pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
99-
100-
101-
python -m pip install coverage
102-
python -m pip install diff-cover
103-
python -m pip install ${fd_wheel_url}
104-
export COVERAGE_FILE=/workspace/FastDeploy/coveragedata/.coverage
105-
export COVERAGE_RCFILE=/workspace/FastDeploy/scripts/.coveragerc
106-
TEST_EXIT_CODE=0
107-
bash scripts/coverage_run.sh || TEST_EXIT_CODE=8
108-
git diff origin/${BASE_REF}..HEAD --unified=0 > diff.txt
109-
echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> exit_code.env
110-
coverage combine coveragedata/
111-
coverage xml -o python_coverage_all.xml
112-
COVERAGE_EXIT_CODE=0
113-
diff-cover python_coverage_all.xml --diff-file=diff.txt --fail-under=80 --json-report diff_coverage.json || COVERAGE_EXIT_CODE=9
114-
echo "COVERAGE_EXIT_CODE=${COVERAGE_EXIT_CODE}" >> exit_code.env
115-
python scripts/generate_diff_coverage_xml.py diff.txt python_coverage_all.xml
116-
'
117-
if [ -f FastDeploy/exit_code.env ]; then
118-
cat FastDeploy/exit_code.env >> $GITHUB_ENV
119-
fi
77+
set -x
78+
runner_name="${{ runner.name }}"
79+
CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
80+
DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
81+
DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1)
82+
83+
FLASK_PORT=$((42068 + DEVICE_PORT * 100))
84+
FD_API_PORT=$((42088 + DEVICE_PORT * 100))
85+
FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
86+
FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
87+
echo "Test ENV Parameter:"
88+
echo "========================================================="
89+
echo "FLASK_PORT=${FLASK_PORT}"
90+
echo "FD_API_PORT=${FD_API_PORT}"
91+
echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
92+
echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
93+
echo "DEVICES=${DEVICES}"
94+
echo "========================================================="
95+
96+
CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
97+
echo "CACHE_DIR is set to ${CACHE_DIR}"
98+
if [ ! -f "${CACHE_DIR}/gitconfig" ]; then
99+
touch "${CACHE_DIR}/gitconfig"
100+
fi
101+
102+
PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
103+
LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
104+
echo "==== LOG_FILE is ${LOG_FILE} ===="
105+
106+
echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE
107+
108+
for port in "${PORTS[@]}"; do
109+
PIDS=$(lsof -t -i :$port || true)
110+
if [ -n "$PIDS" ]; then
111+
echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
112+
echo "$PIDS" | xargs -r kill -9
113+
echo "Port $port cleared" | tee -a $LOG_FILE
114+
else
115+
echo "Port $port is free" | tee -a $LOG_FILE
116+
fi
117+
done
118+
119+
echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE
120+
121+
docker run --rm --net=host \
122+
--cap-add=SYS_PTRACE --shm-size=64G \
123+
-v $(pwd):/workspace -w /workspace \
124+
-v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
125+
-v "${CACHE_DIR}/.cache:/root/.cache" \
126+
-v "${CACHE_DIR}/ConfigDir:/root/.config" \
127+
-v "${MODEL_CACHE_DIR}:/ModelData:ro" \
128+
-e "MODEL_PATH=/ModelData" \
129+
-e "FD_API_PORT=${FD_API_PORT}" \
130+
-e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
131+
-e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
132+
-e "FLASK_PORT=${FLASK_PORT}" \
133+
-e TZ="Asia/Shanghai" \
134+
-e "fd_wheel_url=${fd_wheel_url}" \
135+
-e "BASE_REF=${BASE_REF}" \
136+
--gpus "\"device=${DEVICES}\"" ${docker_image} /bin/bash -c '
137+
138+
git config --global --add safe.directory /workspace/FastDeploy
139+
cd FastDeploy
140+
python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
141+
142+
pip config set global.extra-index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
143+
144+
python -m pip install coverage
145+
python -m pip install diff-cover
146+
python -m pip install ${fd_wheel_url}
147+
if [ -d "test/plugins" ]; then
148+
cd test/plugins
149+
python setup.py install
150+
cd ../..
151+
else
152+
echo "Warning: test/plugins directory not found, skipping setup.py install"
153+
fi
154+
export COVERAGE_FILE=/workspace/FastDeploy/coveragedata/.coverage
155+
export COVERAGE_RCFILE=/workspace/FastDeploy/scripts/.coveragerc
156+
TEST_EXIT_CODE=0
157+
bash scripts/coverage_run.sh || TEST_EXIT_CODE=8
158+
git diff origin/${BASE_REF}..HEAD --unified=0 > diff.txt
159+
echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> exit_code.env
160+
coverage combine coveragedata/
161+
coverage xml -o python_coverage_all.xml
162+
COVERAGE_EXIT_CODE=0
163+
diff-cover python_coverage_all.xml --diff-file=diff.txt --fail-under=80 --json-report diff_coverage.json || COVERAGE_EXIT_CODE=9
164+
echo "COVERAGE_EXIT_CODE=${COVERAGE_EXIT_CODE}" >> exit_code.env
165+
python scripts/generate_diff_coverage_xml.py diff.txt python_coverage_all.xml
166+
'
167+
if [ -f FastDeploy/exit_code.env ]; then
168+
cat FastDeploy/exit_code.env >> $GITHUB_ENV
169+
fi
170+
120171
- name: Upload unit resule and diff coverage to bos
121172
id: cov_upload
122173
shell: bash
@@ -152,32 +203,40 @@ jobs:
152203
echo "unittest_failed_url=${UNIT_TEST_RESULT_URL}" >> $GITHUB_OUTPUT
153204
echo "unittest_failed_url=${UNIT_TEST_RESULT_URL}" >> $GITHUB_ENV
154205
fi
155-
- name: Determine Unit Succ and whether the coverage rate reaches 80%
206+
- name: Check Unit Test Success
156207
shell: bash
157208
run: |
209+
cd FastDeploy
158210
if [ "$TEST_EXIT_CODE" -eq 8 ]; then
211+
filename=$(basename "$unittest_failed_url")
159212
if [ -z "${unittest_failed_url}" ]; then
160213
echo "No diff unit failed file URL provided."
161214
else
162-
wget ${unittest_failed_url} || echo "Download unittest file failed, but continuing..."
215+
rm -rf "${filename}"
216+
wget -O ${filename} ${unittest_failed_url} || echo "Download unittest file failed, but continuing..."
163217
fi
164218
echo "Unit tests failed (exit code 8)"
165-
filename=$(basename "$unittest_failed_url")
166219
if [ -f "${filename}" ];then
167220
echo "Failed test cases:"
168221
cat "${filename}"
169222
fi
170223
exit "$TEST_EXIT_CODE"
171224
fi
225+
echo "All tests passed"
172226
227+
- name: Verify Code Coverage Threshold (80%)
228+
shell: bash
229+
run: |
230+
cd FastDeploy
173231
if [ "$COVERAGE_EXIT_CODE" -eq 9 ]; then
174232
echo "Coverage generation failed (exit code 9)"
233+
filename=$(basename "$diff_cov_result_json_url")
175234
if [ -z "${diff_cov_result_json_url}" ]; then
176235
echo "No diff cov result file URL provided."
177236
else
178-
wget ${diff_cov_result_json_url} || echo "Download cov json file failed, but continuing..."
237+
rm -rf "${filename}"
238+
wget -O ${filename} ${diff_cov_result_json_url} || echo "Download cov json file failed, but continuing..."
179239
fi
180-
filename=$(basename "$diff_cov_result_json_url")
181240
if [ -f "${filename}" ];then
182241
echo "Failed test cases:"
183242
if command -v jq >/dev/null 2>&1; then
@@ -188,7 +247,7 @@ jobs:
188247
fi
189248
exit "$COVERAGE_EXIT_CODE"
190249
fi
191-
echo "All tests and coverage passed"
250+
echo "coverage passed"
192251
exit 0
193252
194253
diff_coverage_report:

0 commit comments

Comments
 (0)