diff --git a/.github/workflows/_build_linux_fd_router.yml b/.github/workflows/_build_linux_fd_router.yml new file mode 100644 index 00000000000..b600cc2328e --- /dev/null +++ b/.github/workflows/_build_linux_fd_router.yml @@ -0,0 +1,213 @@ +name: FastDeploy Linux GPU FD_ROUTER Build Task +description: "FastDeploy FD_ROUTER build and upload" + +on: + workflow_call: + inputs: + DOCKER_IMAGE: + description: "Build Images" + required: true + type: string + default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-build-cuda129-manylinux" + FASTDEPLOY_ARCHIVE_URL: + description: "URL of the compressed FastDeploy code archive." + required: true + type: string + COMPILE_ARCH: + description: "Build GPU Archs" + required: true + type: string + default: "80,90" + WITH_NIGHTLY_BUILD: + description: "Enable nightly build mode (e.g. add date suffix to version)" + required: false + type: string + default: "OFF" + FD_VERSION: + description: "FastDeploy Package Version" + required: false + type: string + default: "" + PADDLEVERSION: + description: "Paddle Version Build Use" + required: false + type: string + default: "" + PADDLE_WHL_URL: + description: "Paddle Wheel Package URL" + required: false + type: string + default: "" + UPLOAD: + description: "Upload Package" + required: false + type: string + default: "ON" + CACHE_DIR: + description: "Cache Dir Use" + required: false + type: string + default: "" + outputs: + fd_router_path: + description: "Output path of the generated wheel" + value: ${{ jobs.fd-router-build.outputs.fd_router_path }} +jobs: + fd-router-build: + runs-on: [self-hosted, GPU-Build-Cu129] + timeout-minutes: 360 + outputs: + fd_router_path: ${{ steps.set_output.outputs.fd_router_path }} + steps: + - name: Code Prepare + shell: bash + env: + docker_image: ${{ inputs.DOCKER_IMAGE }} + fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }} + IS_PR: ${{ github.event_name == 'pull_request' }} + run: | + set -x + REPO="https://github.com/${{ github.repository }}.git" + FULL_REPO="${{ github.repository }}" + REPO_NAME="${FULL_REPO##*/}" + BASE_BRANCH="${{ github.base_ref }}" + # docker pull ${docker_image} + # Clean the repository directory before starting + docker run --rm --net=host -v $(pwd):/workspace -w /workspace \ + -e "REPO_NAME=${REPO_NAME}" \ + ${docker_image} /bin/bash -c ' + CLEAN_RETRIES=3 + CLEAN_COUNT=0 + + while [ $CLEAN_COUNT -lt $CLEAN_RETRIES ]; do + echo "Attempt $((CLEAN_COUNT+1)) to remove ${REPO_NAME}* ..." + rm -rf "${REPO_NAME}"* || true + sleep 2 + + # Check if anything matching ${REPO_NAME}* still exists + if ! ls "${REPO_NAME}"* >/dev/null 2>&1; then + echo "All ${REPO_NAME}* removed successfully" + break + fi + + CLEAN_COUNT=$((CLEAN_COUNT + 1)) + done + + if ls "${REPO_NAME}"* >/dev/null 2>&1; then + echo "ERROR: Failed to clean ${REPO_NAME}* after multiple attempts" + ls -ld "${REPO_NAME}"* + exit 1 + fi + ' + + wget -q --no-proxy ${fd_archive_url} + tar -xf FastDeploy.tar.gz + rm -rf FastDeploy.tar.gz + cd FastDeploy + git config --global user.name "FastDeployCI" + git config --global user.email "fastdeploy_ci@example.com" + git log -n 3 --oneline + - name: FastDeploy FD_ROUTER Build + shell: bash + env: + docker_image: ${{ inputs.DOCKER_IMAGE }} + compile_arch: ${{ inputs.COMPILE_ARCH }} + fd_version: ${{ inputs.FD_VERSION }} + CACHE_DIR: ${{ inputs.CACHE_DIR }} + BRANCH_REF: ${{ github.ref_name }} + PADDLEVERSION: ${{ inputs.PADDLEVERSION }} + PADDLE_WHL_URL: ${{ inputs.PADDLE_WHL_URL }} + WITH_NIGHTLY_BUILD: ${{ inputs.WITH_NIGHTLY_BUILD }} + run: | + set -x + runner_name="${{ runner.name }}" + CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}') + gpu_id=$(echo "$CARD_ID" | fold -w1 | paste -sd,) + + IFS='/' read -ra parts <<< "${GITHUB_WORKSPACE}" + len=${#parts[@]} + CCACHE_DEFAULT_DIR="/$(IFS=/; echo "${parts[*]:1:$((len-5))}")" + echo "$CCACHE_DEFAULT_DIR" + + CACHE_DIR="${CACHE_DIR:-$CCACHE_DEFAULT_DIR}" + echo "CACHE_DIR is set to ${CACHE_DIR}" + if [ ! -f "${CACHE_DIR}/gitconfig" ]; then + touch "${CACHE_DIR}/gitconfig" + fi + PARENT_DIR=$(dirname "$WORKSPACE") + echo "PARENT_DIR:$PARENT_DIR" + docker run --rm --net=host \ + --cap-add=SYS_PTRACE --privileged --shm-size=64G \ + -v $(pwd):/workspace -w /workspace \ + -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \ + -v "${CACHE_DIR}/.cache:/root/.cache" \ + -v "${CACHE_DIR}/.ccache:/root/.ccache" \ + -v "${CACHE_DIR}/ConfigDir:/root/.config" \ + -e TZ="Asia/Shanghai" \ + -e "COMPILE_ARCH=${compile_arch}" \ + -e "FD_VERSION=${fd_version}" \ + -e "WITH_NIGHTLY_BUILD=${WITH_NIGHTLY_BUILD}" \ + -e "PADDLEVERSION=${PADDLEVERSION}" \ + -e "PADDLE_WHL_URL=${PADDLE_WHL_URL}" \ + -e "BRANCH_REF=${BRANCH_REF}" \ + -e "CCACHE_MAXSIZE=50G" \ + --gpus "\"device=${gpu_id}\"" ${docker_image} /bin/bash -c ' + if [[ -n "${FD_VERSION}" ]]; then + export FASTDEPLOY_VERSION=${FD_VERSION} + echo "Custom FastDeploy version: ${FASTDEPLOY_VERSION}" + fi + + git config --global --add safe.directory /workspace/FastDeploy + chown -R $(whoami) /workspace/FastDeploy + cd FastDeploy + if [[ "${WITH_NIGHTLY_BUILD}" == "ON" ]];then + GIT_COMMIT_TIME=$(git --no-pager show -s --format=%ci HEAD) + DATE_ONLY=$(echo $GIT_COMMIT_TIME | sed "s/ .*//;s/-//g") + echo "Git Commit Time: $GIT_COMMIT_TIME" + echo "Date Only: $DATE_ONLY" + export FASTDEPLOY_VERSION="${FASTDEPLOY_VERSION}.dev${DATE_ONLY}" + fi + cd fastdeploy/golang_router + go --version + + bash ./build.sh + ls /usr/local/bin/fd-router* + cp -r /usr/local/bin/fd-router* ./ + ' + + - name: Package Upload + id: set_output + env: + compile_arch: ${{ inputs.COMPILE_ARCH }} + run: | + set -x + if [[ "${{ github.event_name }}" == "pull_request" ]];then + commit_id=${{ github.event.pull_request.head.sha }} + pr_num=${{ github.event.pull_request.number }} + target_path=paddle-github-action/PR/FastDeploy/${pr_num}/${commit_id} + elif [[ "${{ github.ref_type }}" == "tag" ]]; then + commit_id=${{ github.sha }} + tag_name=${{ github.ref_name }} + target_path=paddle-github-action/TAG/FastDeploy/${tag_name}/${commit_id} + else + commit_id=${{ github.sha }} + branch_name=${{ github.ref_name }} + target_path=paddle-github-action/BRANCH/FastDeploy/${branch_name}/${commit_id} + fi + wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py + push_file=$(realpath bos_tools.py) + python --version + python -m pip install bce-python-sdk==0.9.29 + + cd FastDeploy/fastdeploy/golang_router + if [ ! -f fd-router ]; then + echo "Error: fd-router not found in FastDeploy/fastdeploy/golang_router" + exit 1 + fi + + echo "Found: fd-router" + + python ${push_file} fd-router ${target_path} + target_path_stripped="${target_path#paddle-github-action/}" + FD_ROUTER_PATH=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/fd-router + echo "fd_router_path=${FD_ROUTER_PATH}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/publish_job.yml b/.github/workflows/publish_job.yml index d1fc408beb3..47c194d608e 100644 --- a/.github/workflows/publish_job.yml +++ b/.github/workflows/publish_job.yml @@ -228,6 +228,75 @@ jobs: PADDLEVERSION: ${{ needs.publish_pre_check.outputs.compile_use_paddle_version }} PADDLE_WHL_URL: ${{ needs.publish_pre_check.outputs.compile_use_paddle_whl_url }} + build_fd_router: + name: BUILD_FD_ROUTER + needs: [ clone, publish_pre_check ] + uses: ./.github/workflows/_build_linux_fd_router.yml + with: + DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-build-cuda129-manylinux + FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} + COMPILE_ARCH: "80,90" + WITH_NIGHTLY_BUILD: ${{ needs.publish_pre_check.outputs.with_nightly_build }} + FD_VERSION: ${{ needs.publish_pre_check.outputs.fd_version }} + PADDLEVERSION: ${{ needs.publish_pre_check.outputs.compile_use_paddle_version }} + PADDLE_WHL_URL: ${{ needs.publish_pre_check.outputs.compile_use_paddle_whl_url }} + + ce_upload_fd_router: + environment: CodeSync + name: CE_UPLOAD_FD_ROUTER + needs: build_fd_router + runs-on: ubuntu-latest + env: + AK: ${{ secrets.BOS_AK }} + SK: ${{ secrets.BOS_SK }} + FD_ROUTER_URL: ${{ needs.build_fd_router.outputs.fd_router_path }} + steps: + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Fd-Router Info Show and Upload + if: github.ref_name == 'develop' || github.ref_type == 'tag' + run: | + echo "The fd_router is located at: ${{ needs.build_fd_router.outputs.fd_router_path }}" + wget -q --no-check-certificate ${{ needs.build_fd_router.outputs.fd_router_path }} + filename=$(basename ${{ needs.build_fd_router.outputs.fd_router_path }}) + + commit_id=${{ github.sha }} + + wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py + push_file=$(realpath bos_tools.py) + python -m pip install bce-python-sdk==0.9.29 + + if [[ "${{ github.ref_name }}" == "develop" ]];then + branch_name=${{ github.ref_name }} + target_paths=( + "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/${branch_name}/${commit_id}" + "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/${branch_name}/latest" + ) + elif [[ "${{ github.ref_type }}" == "tag" ]]; then + tag_name=${{ github.ref_name }} + target_paths=( + "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/${tag_name}/${commit_id}" + "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/${tag_name}/latest" + ) + else + echo "Not develop or tag, do nothing" + fi + + for target_path in "${target_paths[@]}"; do + echo "Uploading ${filename} to ${target_path}" + python "${push_file}" "${filename}" "${target_path}" + done + + base_prefix="paddle-qa/" + commit_path_stripped="${target_paths[0]#${base_prefix}}" + latest_path_stripped="${target_paths[1]#${base_prefix}}" + FD_ROUTER_PATH="https://paddle-qa.bj.bcebos.com/${commit_path_stripped}/${filename}" + FD_ROUTER_PATH_LATEST="https://paddle-qa.bj.bcebos.com/${latest_path_stripped}/${filename}" + + echo "commit fd-router url is ${FD_ROUTER_PATH}" + echo "latest fd-router url is ${FD_ROUTER_PATH_LATEST}" + paddle_pypi_upload_sm8090: environment: PaddleSourceUpload name: PADDLE_PYPI_UPLOAD_8090