Skip to content

Commit 1ace375

Browse files
XieYunshenXieYunshen
authored
Optimize CI execution workflow (#3371)
* Optimize CI execution workflow * fix
1 parent be94bdd commit 1ace375

File tree

5 files changed

+257
-185
lines changed

5 files changed

+257
-185
lines changed

‎.github/workflows/_base_test.yml

Lines changed: 106 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -70,96 +70,109 @@ jobs:
7070
CACHE_DIR: ${{ inputs.CACHE_DIR }}
7171
MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }}
7272
run: |
73-
runner_name="${{ runner.name }}"
74-
last_char="${runner_name: -1}"
75-
76-
if [[ "$last_char" =~ [0-7] ]]; then
77-
DEVICES="$last_char"
78-
else
79-
DEVICES="0"
80-
fi
81-
82-
FLASK_PORT=$((42068 + DEVICES * 100))
83-
FD_API_PORT=$((42088 + DEVICES * 100))
84-
FD_ENGINE_QUEUE_PORT=$((42058 + DEVICES * 100))
85-
FD_METRICS_PORT=$((42078 + DEVICES * 100))
86-
echo "Test ENV Parameter:"
87-
echo "========================================================="
88-
echo "FLASK_PORT=${FLASK_PORT}"
89-
echo "FD_API_PORT=${FD_API_PORT}"
90-
echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
91-
echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
92-
echo "DEVICES=${DEVICES}"
93-
echo "========================================================="
94-
95-
CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
96-
echo "CACHE_DIR is set to ${CACHE_DIR}"
97-
if [ ! -f "${CACHE_DIR}/gitconfig" ]; then
98-
touch "${CACHE_DIR}/gitconfig"
99-
fi
100-
if [ ! -d "${MODEL_CACHE_DIR}" ]; then
101-
echo "Error: MODEL_CACHE_DIR '${MODEL_CACHE_DIR}' does not exist."
102-
exit 1
103-
fi
104-
105-
PARENT_DIR=$(dirname "$WORKSPACE")
106-
107-
docker run --rm --ipc=host --pid=host --net=host \
108-
-v $(pwd):/workspace \
109-
-w /workspace \
110-
-e fastdeploy_wheel_url=${fastdeploy_wheel_url} \
111-
-e "FD_API_PORT=${FD_API_PORT}" \
112-
-e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
113-
-e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
114-
-e "FLASK_PORT=${FLASK_PORT}" \
115-
-v "${MODEL_CACHE_DIR}:/MODELDATA" \
116-
-v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
117-
-v "${CACHE_DIR}/.cache:/root/.cache" \
118-
-v "${CACHE_DIR}/ConfigDir:/root/.config" \
119-
-e TZ="Asia/Shanghai" \
120-
--gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -xc '
121-
python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
122-
123-
pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
124-
125-
python -m pip install ${fastdeploy_wheel_url}
126-
python -m pip install pytest
127-
128-
wget https://paddle-qa.bj.bcebos.com/zhengtianyu/tools/llm-deploy-linux-amd64
129-
chmod +x ./llm-deploy-linux-amd64
130-
./llm-deploy-linux-amd64 -python python3.10 \
131-
-model_name ERNIE-4.5-0.3B-Paddle \
132-
-model_path /MODELDATA \
133-
--skip install
134-
135-
git config --global --add safe.directory /workspace/FastDeploy
136-
cd FastDeploy
137-
pushd test/ce/deploy
138-
python3.10 deploy.py > dd.log 2>&1 &
139-
sleep 3
140-
curl -X POST http://0.0.0.0:${FLASK_PORT}/start \
141-
-H "Content-Type: application/json" \
142-
-d "{\"--model\": \"/MODELDATA/ERNIE-4.5-0.3B-Paddle\"}"
143-
144-
curl -X POST http://localhost:${FLASK_PORT}/wait_for_infer?timeout=90
145-
popd
146-
147-
pushd test/ce/server
148-
export URL=http://localhost:${FD_API_PORT}/v1/chat/completions
149-
export TEMPLATE=TOKEN_LOGPROB
150-
TEST_EXIT_CODE=0
151-
python -m pytest -sv test_base_chat.py test_compare_top_logprobs.py test_logprobs.py test_params_boundary.py test_seed_usage.py test_stream.py test_evil_cases.py || TEST_EXIT_CODE=1
152-
curl -X POST http://0.0.0.0:${FLASK_PORT}/switch \
153-
-H "Content-Type: application/json" \
154-
-d "{\"--model\": \"/MODELDATA/ERNIE-4.5-0.3B-Paddle\", \"--early-stop-config\": \"{\\\"enable_early_stop\\\":true, \\\"window_size\\\":6, \\\"threshold\\\":0.93}\"}"
155-
curl -X POST http://localhost:${FLASK_PORT}/wait_for_infer?timeout=90
156-
python -m pytest -sv test_repetition_early_stop.py || TEST_EXIT_CODE=1
157-
popd
158-
echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> /workspace/FastDeploy/exit_code.env
159-
'
160-
if [ -f ./FastDeploy/exit_code.env ]; then
161-
source ./FastDeploy/exit_code.env
162-
cat ./FastDeploy/exit_code.env >> $GITHUB_ENV
163-
fi
164-
echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}"
165-
exit ${TEST_EXIT_CODE}
73+
runner_name="${{ runner.name }}"
74+
CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
75+
DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
76+
DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1)
77+
78+
FLASK_PORT=$((42068 + DEVICE_PORT * 100))
79+
FD_API_PORT=$((42088 + DEVICE_PORT * 100))
80+
FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
81+
FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
82+
echo "Test ENV Parameter:"
83+
echo "========================================================="
84+
echo "FLASK_PORT=${FLASK_PORT}"
85+
echo "FD_API_PORT=${FD_API_PORT}"
86+
echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
87+
echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
88+
echo "DEVICES=${DEVICES}"
89+
echo "========================================================="
90+
91+
CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
92+
echo "CACHE_DIR is set to ${CACHE_DIR}"
93+
if [ ! -f "${CACHE_DIR}/gitconfig" ]; then
94+
touch "${CACHE_DIR}/gitconfig"
95+
fi
96+
if [ ! -d "${MODEL_CACHE_DIR}" ]; then
97+
echo "Error: MODEL_CACHE_DIR '${MODEL_CACHE_DIR}' does not exist."
98+
exit 1
99+
fi
100+
101+
PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
102+
LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
103+
echo "==== LOG_FILE is ${LOG_FILE} ===="
104+
105+
echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE
106+
107+
for port in "${PORTS[@]}"; do
108+
PIDS=$(lsof -t -i :$port || true)
109+
if [ -n "$PIDS" ]; then
110+
echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
111+
echo "$PIDS" | xargs -r kill -9
112+
echo "Port $port cleared" | tee -a $LOG_FILE
113+
else
114+
echo "Port $port is free" | tee -a $LOG_FILE
115+
fi
116+
done
117+
118+
echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE
119+
120+
docker run --rm --ipc=host --pid=host --net=host \
121+
-v $(pwd):/workspace \
122+
-w /workspace \
123+
-e fastdeploy_wheel_url=${fastdeploy_wheel_url} \
124+
-e "FD_API_PORT=${FD_API_PORT}" \
125+
-e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
126+
-e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
127+
-e "FLASK_PORT=${FLASK_PORT}" \
128+
-v "${MODEL_CACHE_DIR}:/MODELDATA" \
129+
-v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
130+
-v "${CACHE_DIR}/.cache:/root/.cache" \
131+
-v "${CACHE_DIR}/ConfigDir:/root/.config" \
132+
-e TZ="Asia/Shanghai" \
133+
--gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -xc '
134+
python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
135+
136+
pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
137+
138+
python -m pip install ${fastdeploy_wheel_url}
139+
python -m pip install pytest
140+
141+
wget https://paddle-qa.bj.bcebos.com/zhengtianyu/tools/llm-deploy-linux-amd64
142+
chmod +x ./llm-deploy-linux-amd64
143+
./llm-deploy-linux-amd64 -python python3.10 \
144+
-model_name ERNIE-4.5-0.3B-Paddle \
145+
-model_path /MODELDATA \
146+
--skip install
147+
148+
git config --global --add safe.directory /workspace/FastDeploy
149+
cd FastDeploy
150+
pushd test/ce/deploy
151+
python3.10 deploy.py > dd.log 2>&1 &
152+
sleep 3
153+
curl -X POST http://0.0.0.0:${FLASK_PORT}/start \
154+
-H "Content-Type: application/json" \
155+
-d "{\"--model\": \"/MODELDATA/ERNIE-4.5-0.3B-Paddle\"}"
156+
157+
curl -X POST http://localhost:${FLASK_PORT}/wait_for_infer?timeout=90
158+
popd
159+
160+
pushd test/ce/server
161+
export URL=http://localhost:${FD_API_PORT}/v1/chat/completions
162+
export TEMPLATE=TOKEN_LOGPROB
163+
TEST_EXIT_CODE=0
164+
python -m pytest -sv test_base_chat.py test_compare_top_logprobs.py test_logprobs.py test_params_boundary.py test_seed_usage.py test_stream.py test_evil_cases.py || TEST_EXIT_CODE=1
165+
curl -X POST http://0.0.0.0:${FLASK_PORT}/switch \
166+
-H "Content-Type: application/json" \
167+
-d "{\"--model\": \"/MODELDATA/ERNIE-4.5-0.3B-Paddle\", \"--early-stop-config\": \"{\\\"enable_early_stop\\\":true, \\\"window_size\\\":6, \\\"threshold\\\":0.93}\"}"
168+
curl -X POST http://localhost:${FLASK_PORT}/wait_for_infer?timeout=90
169+
python -m pytest -sv test_repetition_early_stop.py || TEST_EXIT_CODE=1
170+
popd
171+
echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> /workspace/FastDeploy/exit_code.env
172+
'
173+
if [ -f ./FastDeploy/exit_code.env ]; then
174+
source ./FastDeploy/exit_code.env
175+
cat ./FastDeploy/exit_code.env >> $GITHUB_ENV
176+
fi
177+
echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}"
178+
exit ${TEST_EXIT_CODE}

‎.github/workflows/_logprob_test_linux.yml

Lines changed: 25 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -62,18 +62,14 @@ jobs:
6262
MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }}
6363
run: |
6464
runner_name="${{ runner.name }}"
65-
last_char="${runner_name: -1}"
65+
CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
66+
DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
67+
DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1)
6668
67-
if [[ "$last_char" =~ [0-7] ]]; then
68-
DEVICES="$last_char"
69-
else
70-
DEVICES="0"
71-
fi
72-
73-
FLASK_PORT=$((42068 + DEVICES * 100))
74-
FD_API_PORT=$((42088 + DEVICES * 100))
75-
FD_ENGINE_QUEUE_PORT=$((42058 + DEVICES * 100))
76-
FD_METRICS_PORT=$((42078 + DEVICES * 100))
69+
FLASK_PORT=$((42068 + DEVICE_PORT * 100))
70+
FD_API_PORT=$((42088 + DEVICE_PORT * 100))
71+
FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100))
72+
FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100))
7773
echo "Test ENV Parameter:"
7874
echo "========================================================="
7975
echo "FLASK_PORT=${FLASK_PORT}"
@@ -93,9 +89,24 @@ jobs:
9389
exit 1
9490
fi
9591
96-
PARENT_DIR=$(dirname "$WORKSPACE")
97-
unset http_proxy
98-
unset https_proxy
92+
PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
93+
LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
94+
echo "==== LOG_FILE is ${LOG_FILE} ===="
95+
96+
echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE
97+
98+
for port in "${PORTS[@]}"; do
99+
PIDS=$(lsof -t -i :$port || true)
100+
if [ -n "$PIDS" ]; then
101+
echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
102+
echo "$PIDS" | xargs -r kill -9
103+
echo "Port $port cleared" | tee -a $LOG_FILE
104+
else
105+
echo "Port $port is free" | tee -a $LOG_FILE
106+
fi
107+
done
108+
109+
echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE
99110
100111
docker run --rm --ipc=host --pid=host --net=host \
101112
-v $(pwd):/workspace \

‎.github/workflows/_pre_ce_test.yml

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,25 @@ jobs:
9999
touch "${CACHE_DIR}/gitconfig"
100100
fi
101101
102+
PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT)
103+
LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
104+
echo "==== LOG_FILE is ${LOG_FILE} ===="
105+
106+
echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE
107+
108+
for port in "${PORTS[@]}"; do
109+
PIDS=$(lsof -t -i :$port || true)
110+
if [ -n "$PIDS" ]; then
111+
echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
112+
echo "$PIDS" | xargs -r kill -9
113+
echo "Port $port cleared" | tee -a $LOG_FILE
114+
else
115+
echo "Port $port is free" | tee -a $LOG_FILE
116+
fi
117+
done
118+
119+
echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE
120+
102121
docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
103122
-v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
104123
-v "${CACHE_DIR}/.cache:/root/.cache" \
@@ -115,8 +134,5 @@ jobs:
115134
cd FastDeploy
116135
python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
117136
python -m pip install ${fd_wheel_url}
118-
for port in $FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT; do
119-
lsof -t -i :$port | xargs -r kill -9 || true
120-
done
121137
bash scripts/run_pre_ce.sh
122138
'

0 commit comments

Comments
 (0)