publish_job.yml

name: Publish Job

on:
  workflow_dispatch:
  schedule:
    - cron: '0 18 * * *'   # 2:00 AM China Standard Time (UTC+8)
  push:
    # branches:
    #   - develop
    tags:
      - '*'

permissions: read-all

concurrency:
  group: Publish-Job-${{ github.ref }}-${{ github.sha }}
  cancel-in-progress: true


jobs:
  publish_pre_check:
    runs-on: ubuntu-latest
    if: |
      github.event.repository.fork == false &&
      (
        (github.event_name == 'schedule' && github.ref_name == 'develop') ||
        (github.event_name == 'push' && github.ref_type == 'tag') ||
        ((github.event_name == 'workflow_dispatch') &&
          (github.ref_name == 'develop' || github.ref_type == 'tag'))
      )
    env:
      TAG_VERSION_MAPPINGS: ${{ vars.TAG_VERSION_MAPPINGS }}
      FD_VERSION_DEV: ${{ vars.FD_VERSION_DEV }}
      COMPILE_USE_PADDLE_WHL_URL_MAPPINGS: ${{ vars.COMPILE_USE_PADDLE_WHL_URL_MAPPINGS }}
    outputs:
      compile_use_paddle_version: ${{ steps.set_output.outputs.compile_use_paddle_version }}
      compile_continue: ${{ steps.set_output.outputs.compile_continue }}
      fd_version: ${{ steps.set_output.outputs.fd_version }}
      with_nightly_build: ${{ steps.set_output.outputs.with_nightly_build }}
      compile_use_paddle_whl_url: ${{ steps.set_output.outputs.compile_use_paddle_whl_url }}

    steps:
      - name: Get tag version
        if: github.ref_type == 'tag'
        run: |
          TAG_NAME="${GITHUB_REF##*/}"       # 提取 tag 名称，比如 v2.1.0
          TAG_VERSION="${TAG_NAME#v}"            # 去掉前缀 v
          echo "FD_VERSION=$TAG_VERSION" >> $GITHUB_ENV

      - name: Check FD version to Paddle version mapping
        if: github.ref_type == 'tag'
        env:
          TARGET_FD: ${{ env.FD_VERSION }}
        run: |
          FOUND_PADDLE=""
          # 遍历映射
          for pair in $(echo $TAG_VERSION_MAPPINGS | tr ';' ' '); do
            fd=$(echo "$pair" | cut -d',' -f1)
            paddle=$(echo "$pair" | cut -d',' -f2)
            if [[ "$fd" == "$TARGET_FD" ]]; then
              FOUND_PADDLE="$paddle"
              break
            fi
          done

          if [[ -z "$FOUND_PADDLE" ]]; then
            echo "No Paddle version found for FD $TARGET_FD"
          else
            echo "FD $TARGET_FD maps to Paddle $FOUND_PADDLE"
            echo "PADDLE_VERSION=$FOUND_PADDLE" >> $GITHUB_ENV
          fi
      - name: Set Version
        id: set_output
        env:
          PADDLE_VERSION: ${{ env.PADDLE_VERSION }}
          FD_VERSION: ${{ env.FD_VERSION }}
        run: |
          if [[ "${{ github.ref_type }}" == "tag" ]]; then
            if [[ -z "$PADDLE_VERSION" ]]; then
              compile_continue=false
            else
              compile_use_paddle_version=$PADDLE_VERSION
              compile_continue=true
            fi
            fd_version=$FD_VERSION
          fi
          if [[ "${{ github.ref_name }}" == "develop" ]];then
            compile_continue=true
            compile_use_paddle_version=""
            fd_version=${FD_VERSION_DEV}
            with_nightly_build=ON
          fi
          # Todo
          # 通过变量COMPILE_USE_PADDLE_WHL_URL_MAPPINGS中的映射关系,决定是否是安装指定版本的Paddle还是直接安装URL
          for pair in $(echo $COMPILE_USE_PADDLE_WHL_URL_MAPPINGS | tr ';' ' '); do
            branch=$(echo "$pair" | cut -d',' -f1)
            paddle_whl_url=$(echo "$pair" | cut -d',' -f2)
            if [[ "$branch" == "${{ github.ref_name }}" ]]; then
              FOUND_PADDLE_URL="$paddle_whl_url"
              echo "compile_use_paddle_whl_url=${FOUND_PADDLE_URL}" >> $GITHUB_OUTPUT
              compile_continue=true
              break
            fi
          done
          echo "compile_continue=${compile_continue}" >> $GITHUB_OUTPUT
          echo "compile_use_paddle_version=${compile_use_paddle_version}" >> $GITHUB_OUTPUT
          echo "fd_version=${fd_version}" >> $GITHUB_OUTPUT
          echo "with_nightly_build=${with_nightly_build:-OFF}" >> $GITHUB_OUTPUT

  print_publish_pre_check_outputs:
    runs-on: ubuntu-latest
    needs: publish_pre_check
    steps:
      - name: Print outputs as JSON
        run: |
          echo '${{ toJSON(needs.publish_pre_check.outputs) }}'

  clone:
    environment: CodeSync
    name: FD-Clone-Linux
    runs-on: ubuntu-latest
    needs: publish_pre_check
    if: ${{ needs.publish_pre_check.outputs.compile_continue == 'true' }}
    outputs:
      repo_archive_url: ${{ steps.set_output.outputs.repo_archive_url }}
    steps:
      - name: Clone FastDeploy
        uses: actions/checkout@v4
        with:
          ref: ${{ github.ref_name }}
          submodules: 'recursive'
          fetch-depth: 1000

      - name: Python Setup
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: Code Info Show and Upload
        id: set_output
        env:
          AK: ${{ secrets.BOS_AK }}
          SK: ${{ secrets.BOS_SK }}
        run: |
          git config --unset http.https://github.com/.extraheader
          git submodule foreach --recursive sh -c "git config --local --unset-all 'http.https://github.com/.extraheader'"
          git submodule foreach --recursive sh -c "git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'"
          echo "Current HEAD Log:"
          git log --oneline -n 5
          ls
          cd ..
          tar -zcf FastDeploy.tar.gz FastDeploy
          if [[ "${{ github.ref_type }}" == "tag" ]]; then
            commit_id=${{ github.sha }}
            tag_name=${{ github.ref_name }}
            target_path=paddle-qa/TAG/FastDeploy/${tag_name}/${commit_id}
          else
            commit_id=${{ github.sha }}
            branch_name=${{ github.ref_name }}
            target_path=paddle-qa/BRANCH/FastDeploy/${branch_name}/${commit_id}
          fi
          wget  -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
          push_file=$(realpath bos_tools.py)
          python -m pip install bce-python-sdk==0.9.29
          ls
          python ${push_file} FastDeploy.tar.gz ${target_path}
          target_path_stripped="${target_path#paddle-qa/}"
          REPO_ARCHIVE_URL=https://paddle-qa.bj.bcebos.com/${target_path_stripped}/FastDeploy.tar.gz
          echo "repo_archive_url=${REPO_ARCHIVE_URL}" >> $GITHUB_OUTPUT

  resultshow:
    name: Show Code Archive Output
    needs: clone
    runs-on: ubuntu-latest
    steps:
      - name: Print wheel path
        run: |
          echo "The code archive is located at: ${{ needs.clone.outputs.repo_archive_url }}"

  build_sm8090:
    name: BUILD_SM8090
    needs: [clone, publish_pre_check]
    uses: ./.github/workflows/_build_linux.yml
    with:
      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-build-cuda126-manylinux
      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
      COMPILE_ARCH: "80,90"
      WITH_NIGHTLY_BUILD: ${{ needs.publish_pre_check.outputs.with_nightly_build }}
      FD_VERSION: ${{ needs.publish_pre_check.outputs.fd_version }}
      PADDLEVERSION: ${{ needs.publish_pre_check.outputs.compile_use_paddle_version }}
      PADDLE_WHL_URL: ${{ needs.publish_pre_check.outputs.compile_use_paddle_whl_url }}

  build_sm8689:
    name: BUILD_SM8689
    needs: [clone, publish_pre_check]
    uses: ./.github/workflows/_build_linux.yml
    with:
      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-build-cuda126-manylinux
      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
      COMPILE_ARCH: "86,89"
      WITH_NIGHTLY_BUILD: ${{ needs.publish_pre_check.outputs.with_nightly_build }}
      FD_VERSION: ${{ needs.publish_pre_check.outputs.fd_version }}
      PADDLEVERSION: ${{ needs.publish_pre_check.outputs.compile_use_paddle_version }}
      PADDLE_WHL_URL: ${{ needs.publish_pre_check.outputs.compile_use_paddle_whl_url }}

  build_sm8090_cu129:
    name: BUILD_SM8090_cu129
    needs: [ clone, publish_pre_check ]
    uses: ./.github/workflows/_build_linux_cu129.yml
    with:
      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-build-cuda129-manylinux
      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
      COMPILE_ARCH: "80,90"
      WITH_NIGHTLY_BUILD: ${{ needs.publish_pre_check.outputs.with_nightly_build }}
      FD_VERSION: ${{ needs.publish_pre_check.outputs.fd_version }}
      PADDLEVERSION: ${{ needs.publish_pre_check.outputs.compile_use_paddle_version }}
      PADDLE_WHL_URL: ${{ needs.publish_pre_check.outputs.compile_use_paddle_whl_url }}

  build_sm8689_cu129:
    name: BUILD_SM8689_cu129
    needs: [ clone, publish_pre_check ]
    uses: ./.github/workflows/_build_linux_cu129.yml
    with:
      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-build-cuda129-manylinux
      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
      COMPILE_ARCH: "86,89"
      WITH_NIGHTLY_BUILD: ${{ needs.publish_pre_check.outputs.with_nightly_build }}
      FD_VERSION: ${{ needs.publish_pre_check.outputs.fd_version }}
      PADDLEVERSION: ${{ needs.publish_pre_check.outputs.compile_use_paddle_version }}
      PADDLE_WHL_URL: ${{ needs.publish_pre_check.outputs.compile_use_paddle_whl_url }}

  build_fd_router:
    name: BUILD_FD_ROUTER
    needs: [ clone, publish_pre_check ]
    uses: ./.github/workflows/_build_linux_fd_router.yml
    with:
      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-build-cuda129-manylinux
      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
      COMPILE_ARCH: "80,90"
      WITH_NIGHTLY_BUILD: ${{ needs.publish_pre_check.outputs.with_nightly_build }}
      FD_VERSION: ${{ needs.publish_pre_check.outputs.fd_version }}
      PADDLEVERSION: ${{ needs.publish_pre_check.outputs.compile_use_paddle_version }}
      PADDLE_WHL_URL: ${{ needs.publish_pre_check.outputs.compile_use_paddle_whl_url }}

  ce_upload_fd_router:
    environment: CodeSync
    name: CE_UPLOAD_FD_ROUTER
    needs: build_fd_router
    runs-on: ubuntu-latest
    env:
      AK: ${{ secrets.BOS_AK }}
      SK: ${{ secrets.BOS_SK }}
      FD_ROUTER_URL: ${{ needs.build_fd_router.outputs.fd_router_path }}
    steps:
      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: Fd-Router Info Show and Upload
        if: github.ref_name == 'develop' || github.ref_type == 'tag'
        run: |
          echo "The fd_router is located at: ${{ needs.build_fd_router.outputs.fd_router_path }}"
          wget -q --no-check-certificate ${{ needs.build_fd_router.outputs.fd_router_path }}
          filename=$(basename ${{ needs.build_fd_router.outputs.fd_router_path }})

          commit_id=${{ github.sha }}

          wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
          push_file=$(realpath bos_tools.py)
          python -m pip install bce-python-sdk==0.9.29

          if [[ "${{ github.ref_name }}" == "develop" ]];then
            branch_name=${{ github.ref_name }}
            target_paths=(
              "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/${branch_name}/${commit_id}"
              "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/${branch_name}/latest"
            )
          elif [[ "${{ github.ref_type }}" == "tag" ]]; then
            tag_name=${{ github.ref_name }}
            target_paths=(
              "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/${tag_name}/${commit_id}"
              "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/${tag_name}/latest"
            )
          else
            echo "Not develop or tag, do nothing"
          fi

          for target_path in "${target_paths[@]}"; do
            echo "Uploading ${filename} to ${target_path}"
            python "${push_file}" "${filename}" "${target_path}"
          done

          base_prefix="paddle-qa/"
          commit_path_stripped="${target_paths[0]#${base_prefix}}"
          latest_path_stripped="${target_paths[1]#${base_prefix}}"
          FD_ROUTER_PATH="https://paddle-qa.bj.bcebos.com/${commit_path_stripped}/${filename}"
          FD_ROUTER_PATH_LATEST="https://paddle-qa.bj.bcebos.com/${latest_path_stripped}/${filename}"

          echo "commit fd-router url is ${FD_ROUTER_PATH}"
          echo "latest fd-router url is ${FD_ROUTER_PATH_LATEST}"

  paddle_pypi_upload_sm8090:
    environment: PaddleSourceUpload
    name: PADDLE_PYPI_UPLOAD_8090
    needs: build_sm8090
    runs-on: ubuntu-latest
    env:
      AK: ${{ secrets.BOS_AK }}
      SK: ${{ secrets.BOS_SK }}
      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
      COMPILE_ARCH: "80,90"
    steps:
      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: Wheel Info Show and Upload
        if: github.ref_name == 'develop' || github.ref_type == 'tag'
        run: |
          echo "The wheel is located at: ${FASTDEPLOY_WHEEL_URL}"
          wget -q --no-check-certificate ${FASTDEPLOY_WHEEL_URL}
          filename=$(basename ${FASTDEPLOY_WHEEL_URL})
          if [[ "${{ github.ref_name }}" == "develop" ]];then
            target_path=paddle-whl/nightly/fastdeploy-gpu-${COMPILE_ARCH//,/_}/fastdeploy-gpu
          elif [[ "${{ github.ref_type }}" == "tag" ]]; then
            target_path=paddle-whl/stable/fastdeploy-gpu-${COMPILE_ARCH//,/_}/fastdeploy-gpu
          else
            echo "Not develop or tag, do nothing"
          fi
          wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
          push_file=$(realpath bos_tools.py)
          python -m pip install bce-python-sdk==0.9.29
          ls
          python ${push_file} ${filename} ${target_path}

  paddle_pypi_upload_sm8689:
    environment: PaddleSourceUpload
    name: PADDLE_PYPI_UPLOAD_8689
    needs: build_sm8689
    runs-on: ubuntu-latest
    env:
      AK: ${{ secrets.BOS_AK }}
      SK: ${{ secrets.BOS_SK }}
      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8689.outputs.wheel_path }}
      COMPILE_ARCH: "86,89"
    steps:
      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: Wheel Info Show and Upload
        if: github.ref_name == 'develop' || github.ref_type == 'tag'
        run: |
          echo "The wheel is located at: ${FASTDEPLOY_WHEEL_URL}"
          wget -q --no-check-certificate ${FASTDEPLOY_WHEEL_URL}
          filename=$(basename ${FASTDEPLOY_WHEEL_URL})
          if [[ "${{ github.ref_name }}" == "develop" ]];then
            target_path=paddle-whl/nightly/fastdeploy-gpu-${COMPILE_ARCH//,/_}/fastdeploy-gpu
          elif [[ "${{ github.ref_type }}" == "tag" ]]; then
            target_path=paddle-whl/stable/fastdeploy-gpu-${COMPILE_ARCH//,/_}/fastdeploy-gpu
          else
            echo "Not develop or tag, do nothing"
          fi
          wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
          push_file=$(realpath bos_tools.py)
          python -m pip install bce-python-sdk==0.9.29
          ls
          python ${push_file} ${filename} ${target_path}

  images_build:
    name: Run FD Image Build
    needs: [clone, publish_pre_check, build_sm8090]
    runs-on: [self-hosted, Docker-Build]
    if: |
      github.event.repository.fork == false &&
      (
        (github.event_name == 'push' && github.ref_type == 'tag') ||
        (github.event_name == 'workflow_dispatch' && github.ref_type == 'tag')
      )
    env:
      FD_VERSION: ${{ needs.publish_pre_check.outputs.fd_version }}
      PADDLEVERSION: ${{ needs.publish_pre_check.outputs.compile_use_paddle_version }}
      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
    steps:
      - name: Images Build
        shell: bash
        env:
          docker_image: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
        run: |
          set -x
          FULL_REPO="${{ github.repository }}"
          REPO_NAME="${FULL_REPO##*/}"
          fd_archive_url="${{ needs.clone.outputs.repo_archive_url }}"

          # Clean the repository directory before starting
          docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
          -e "REPO_NAME=${REPO_NAME}" \
          ${docker_image} /bin/bash -c '
            if [ -d ${REPO_NAME} ]; then
              echo "Directory ${REPO_NAME} exists, removing it..."
              rm -rf ${REPO_NAME}*
            fi
          '
          wget -q --no-proxy ${fd_archive_url}
          tar -xf FastDeploy.tar.gz
          rm -rf FastDeploy.tar.gz
          cd FastDeploy
          git config --global user.name "FastDeployCI"
          git config --global user.email "fastdeploy_ci@example.com"
          git log -n 3 --oneline

          PRODUCT_NAME=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/fastdeploy-cuda-12.6:${FD_VERSION}
          docker build --no-cache -t ${PRODUCT_NAME} -f Dockerfile.gpu . \
            --network host \
            --build-arg PADDLE_VERSION=${PADDLEVERSION} \
            --build-arg FD_VERSION=${FD_VERSION}

          docker push ${PRODUCT_NAME}

  unittest_coverage:
    name: Run FastDeploy Unit Tests and Coverage
    needs: [clone,build_sm8090]
    uses: ./.github/workflows/_unit_test_coverage.yml
    with:
      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
      MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"
    secrets:
      github-token: ${{ secrets.GITHUB_TOKEN }}

  logprob_test:
    name: Run FastDeploy LogProb Tests
    needs: [build_sm8090]
    uses: ./.github/workflows/_logprob_test_linux.yml
    with:
      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
      PADDLETEST_ARCHIVE_URL: "https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz"
      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
      MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"

  pre_ce_test:
    name: Extracted partial CE model tasks to run in CI.
    needs: [clone,build_sm8090]
    uses: ./.github/workflows/_pre_ce_test.yml
    with:
      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
      MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"

  base_test:
    name: Run Base Tests
    needs: [clone,build_sm8090]
    uses: ./.github/workflows/_base_test.yml
    with:
      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
      MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"

  accuracy_test:
    name: Run Accuracy Tests
    needs: [clone,build_sm8090]
    uses: ./.github/workflows/_accuracy_test.yml
    with:
      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
      MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"

  stable_test:
    name: Run Stable Tests
    needs: [clone,build_sm8090]
    uses: ./.github/workflows/_stable_test.yml
    with:
      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
      MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData"