diff --git a/.github/workflows/_pre_ce_test.yml b/.github/workflows/_pre_ce_test.yml index 91368dfda1..1f61ef7869 100644 --- a/.github/workflows/_pre_ce_test.yml +++ b/.github/workflows/_pre_ce_test.yml @@ -21,6 +21,11 @@ on: required: false type: string default: "" + MODEL_CACHE_DIR: + description: "Cache Dir Use" + required: false + type: string + default: "" concurrency: group: ${{ github.event.pull_request.number }} @@ -28,7 +33,7 @@ concurrency: jobs: run_ce_cases: - runs-on: [self-hosted, GPU-L20-4Card] + runs-on: [self-hosted, PRE_CE_RUN_2Card] steps: - name: Print current runner name run: | @@ -67,37 +72,51 @@ jobs: env: docker_image: ${{ inputs.DOCKER_IMAGE }} fd_wheel_url: ${{ inputs.FASTDEPLOY_WHEEL_URL }} + CACHE_DIR: ${{ inputs.CACHE_DIR }} + MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }} run: | runner_name="${{ runner.name }}" - last_char="${runner_name: -1}" + CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}') + DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,) + DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1) + + FLASK_PORT=$((42068 + DEVICE_PORT * 100)) + FD_API_PORT=$((42088 + DEVICE_PORT * 100)) + FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100)) + FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100)) + echo "Test ENV Parameter:" + echo "=========================================================" + echo "FLASK_PORT=${FLASK_PORT}" + echo "FD_API_PORT=${FD_API_PORT}" + echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" + echo "FD_METRICS_PORT=${FD_METRICS_PORT}" + echo "DEVICES=${DEVICES}" + echo "=========================================================" - if [ "${last_char}" = "1" ]; then - gpu_id=2 - DEVICES="2,3" - else - gpu_id=0 - DEVICES="0,1" + CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}" + echo "CACHE_DIR is set to ${CACHE_DIR}" + if [ ! -f "${CACHE_DIR}/gitconfig" ]; then + touch "${CACHE_DIR}/gitconfig" fi - FD_API_PORT=$((9180 + gpu_id * 100)) - FD_ENGINE_QUEUE_PORT=$((9150 + gpu_id * 100)) - FD_METRICS_PORT=$((9170 + gpu_id * 100)) - PARENT_DIR=$(dirname "$WORKSPACE") - echo "PARENT_DIR:$PARENT_DIR" docker run --rm --net=host -v $(pwd):/workspace -w /workspace \ - -v "/ssd4/GithubActions/gitconfig:/etc/gitconfig:ro" \ - -v "/ssd4/GithubActions/ModelData:/ModelData:ro" \ - -v "/ssd4/GithubActions/CacheDir:/root/.cache" \ - -v "/ssd4/GithubActions/ConfigDir:/root/.config" \ + -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \ + -v "${CACHE_DIR}/.cache:/root/.cache" \ + -v "${CACHE_DIR}/ConfigDir:/root/.config" \ + -v "${MODEL_CACHE_DIR}:/ModelData:ro" \ -e "MODEL_PATH=/ModelData" \ -e "FD_API_PORT=${FD_API_PORT}" \ -e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \ -e "FD_METRICS_PORT=${FD_METRICS_PORT}" \ + -e "FLASK_PORT=${FLASK_PORT}" \ -e "fd_wheel_url=${fd_wheel_url}" \ - --gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -c ' + --gpus "\"device=${DEVICES}\"" ${docker_image} /bin/bash -c ' git config --global --add safe.directory /workspace/FastDeploy cd FastDeploy python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/ python -m pip install ${fd_wheel_url} + for port in $FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT; do + lsof -t -i :$port | xargs -r kill -9 || true + done bash scripts/run_pre_ce.sh ' diff --git a/.github/workflows/_unit_test_coverage.yml b/.github/workflows/_unit_test_coverage.yml index b959fff17f..130dbf74b4 100644 --- a/.github/workflows/_unit_test_coverage.yml +++ b/.github/workflows/_unit_test_coverage.yml @@ -22,6 +22,11 @@ on: required: false type: string default: "" + MODEL_CACHE_DIR: + description: "Cache Dir Use" + required: false + type: string + default: "" jobs: run_tests_with_coverage: @@ -67,11 +72,26 @@ jobs: fd_wheel_url: ${{ inputs.FASTDEPLOY_WHEEL_URL }} CACHE_DIR: ${{ inputs.CACHE_DIR }} BASE_REF: ${{ github.event.pull_request.base.ref }} + MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }} run: | set -x runner_name="${{ runner.name }}" CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}') - gpu_id=$(echo "$CARD_ID" | fold -w1 | paste -sd,) + DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,) + DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1) + + FLASK_PORT=$((42068 + DEVICE_PORT * 100)) + FD_API_PORT=$((42088 + DEVICE_PORT * 100)) + FD_ENGINE_QUEUE_PORT=$((42058 + DEVICE_PORT * 100)) + FD_METRICS_PORT=$((42078 + DEVICE_PORT * 100)) + echo "Test ENV Parameter:" + echo "=========================================================" + echo "FLASK_PORT=${FLASK_PORT}" + echo "FD_API_PORT=${FD_API_PORT}" + echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" + echo "FD_METRICS_PORT=${FD_METRICS_PORT}" + echo "DEVICES=${DEVICES}" + echo "=========================================================" CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}" echo "CACHE_DIR is set to ${CACHE_DIR}" @@ -86,17 +106,22 @@ jobs: -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \ -v "${CACHE_DIR}/.cache:/root/.cache" \ -v "${CACHE_DIR}/ConfigDir:/root/.config" \ + -v "${MODEL_CACHE_DIR}:/ModelData:ro" \ + -e "MODEL_PATH=/ModelData" \ + -e "FD_API_PORT=${FD_API_PORT}" \ + -e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \ + -e "FD_METRICS_PORT=${FD_METRICS_PORT}" \ + -e "FLASK_PORT=${FLASK_PORT}" \ -e TZ="Asia/Shanghai" \ -e "fd_wheel_url=${fd_wheel_url}" \ -e "BASE_REF=${BASE_REF}" \ - --gpus "\"device=${gpu_id}\"" ${docker_image} /bin/bash -c ' + --gpus "\"device=${DEVICES}\"" ${docker_image} /bin/bash -c ' git config --global --add safe.directory /workspace/FastDeploy cd FastDeploy python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/ - pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple - + pip config set global.extra-index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple python -m pip install coverage python -m pip install diff-cover @@ -152,32 +177,40 @@ jobs: echo "unittest_failed_url=${UNIT_TEST_RESULT_URL}" >> $GITHUB_OUTPUT echo "unittest_failed_url=${UNIT_TEST_RESULT_URL}" >> $GITHUB_ENV fi - - name: Determine Unit Succ and whether the coverage rate reaches 80% + - name: Check Unit Test Success shell: bash run: | + cd FastDeploy if [ "$TEST_EXIT_CODE" -eq 8 ]; then + filename=$(basename "$unittest_failed_url") if [ -z "${unittest_failed_url}" ]; then echo "No diff unit failed file URL provided." else - wget ${unittest_failed_url} || echo "Download unittest file failed, but continuing..." + rm -rf "${filename}" + wget -O ${filename} ${unittest_failed_url} || echo "Download unittest file failed, but continuing..." fi echo "Unit tests failed (exit code 8)" - filename=$(basename "$unittest_failed_url") if [ -f "${filename}" ];then echo "Failed test cases:" cat "${filename}" fi exit "$TEST_EXIT_CODE" fi + echo "All tests passed" + - name: Verify Code Coverage Threshold (80%) + shell: bash + run: | + cd FastDeploy if [ "$COVERAGE_EXIT_CODE" -eq 9 ]; then echo "Coverage generation failed (exit code 9)" + filename=$(basename "$diff_cov_result_json_url") if [ -z "${diff_cov_result_json_url}" ]; then echo "No diff cov result file URL provided." else - wget ${diff_cov_result_json_url} || echo "Download cov json file failed, but continuing..." + rm -rf "${filename}" + wget -O ${filename} ${diff_cov_result_json_url} || echo "Download cov json file failed, but continuing..." fi - filename=$(basename "$diff_cov_result_json_url") if [ -f "${filename}" ];then echo "Failed test cases:" if command -v jq >/dev/null 2>&1; then @@ -188,7 +221,7 @@ jobs: fi exit "$COVERAGE_EXIT_CODE" fi - echo "All tests and coverage passed" + echo "coverage passed" exit 0 diff_coverage_report: diff --git a/.github/workflows/pr_build_and_test.yml b/.github/workflows/pr_build_and_test.yml index 0123e5a554..73abc2440d 100644 --- a/.github/workflows/pr_build_and_test.yml +++ b/.github/workflows/pr_build_and_test.yml @@ -39,25 +39,27 @@ jobs: needs: [clone,build] uses: ./.github/workflows/_unit_test_coverage.yml with: - DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:cuda126-py310 + DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }} + MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" logprob_test: name: Run FastDeploy LogProb Tests needs: [build] uses: ./.github/workflows/_logprob_test_linux.yml with: - DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:cuda126-py310 + DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate PADDLETEST_ARCHIVE_URL: "https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz" FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }} - MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelCache" + MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" pre_ce_test: name: Extracted partial CE model tasks to run in CI. needs: [clone,build] uses: ./.github/workflows/_pre_ce_test.yml with: - DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:fastdeploy-ciuse-cuda126 + DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }} + MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" diff --git a/test/entrypoints/openai/test_serving_completion.py b/test/entrypoints/openai/test_serving_completion.py index 8d1a4eb663..4c7404a790 100644 --- a/test/entrypoints/openai/test_serving_completion.py +++ b/test/entrypoints/openai/test_serving_completion.py @@ -95,6 +95,7 @@ def test_request_output_to_completion_response(self): model_name=model_name, prompt_batched_token_ids=prompt_batched_token_ids, completion_batched_token_ids=completion_batched_token_ids, + text_after_process_list=["1", "1"], ) assert completion_response.id == request_id