_runs-on-nv-step2.yml

name: runs on nv step 2

on:
  workflow_call:
    inputs:
      runner:
        description: Set up the runner
        type: string
        required: false
        default: "tps-sco-ci"
      deeplink_path:
        description: ci work home
        type: string
        required: false
        default: '/mnt/cache/share/deeplinkci/github/${{ github.repository }}/'
      env_path:
        description: env file path
        type: string
        required: false
        default: '/mnt/cache/share/deeplinkci/github'
      all_coverage:
        description: all coverage
        type: string
        required: true
        default: 'false'
      require_coverage:
        description: input coverage rate
        type: string
        required: false
        default: '0'
jobs:
  Test-Cuda:
    name: Test-dipu-cuda
    runs-on: ${{ inputs.runner }}
    env:
      GETRUNNER: ${{ inputs.runner }}
      DEEPLINK_PATH: ${{ inputs.deeplink_path }}
      ENV_PATH: ${{ inputs.env_path }}
      CUDA_PARTATION: "pat_dev"
      ALL_COVERAGE: ${{ inputs.all_coverage }}
      REQUIRE_COVERAGE: ${{ inputs.require_coverage }}
    steps:
      - name: Run-test
        run: |
          if [[ "${GETRUNNER}" == *sco* ]];then
            set -e
            srun --job-name=need_two_gpus bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu \
            && source ${ENV_PATH}/dipu_env  \
            && bash tests/run_nv_tests.sh"
            if [ "${ALL_COVERAGE}" = "ON" ]; then
            bash /mnt/cache/share/platform/dep/sonar/coverage_DIPU_nv.sh ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda ${GITHUB_RUN_NUMBER} || echo "get coverage fail"
            fi
          else
            ssh SH1424 """
            set -ex
            export USE_COVERAGE=ON
            cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && cd Build-Cuda/dipu
            source ${ENV_PATH}/dipu_env
            srun --job-name=${GITHUB_RUN_NUMBER}_${GITHUB_JOB} --partition=${CUDA_PARTATION} --gres=gpu:2 --cpus-per-task=5 --mem=16G --time=70 sh tests/run_nv_tests.sh
            if [ "${ALL_COVERAGE}" = "ON" ]; then
            bash /mnt/cache/share/platform/dep/sonar/coverage_DIPU_nv.sh ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda ${GITHUB_RUN_NUMBER} || echo "get coverage fail"
            fi
            """
          fi
      - name: increment coverage check
        if: ${{ contains( github.event_name, 'pull_request' ) && contains( github.base_ref, 'main' ) }}
        run: |
          if [[ "${GETRUNNER}" == *sco* ]];then
            set -e
            cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda
            ln -s ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/dipu/third_party/DIOPI/scripts scripts
            source ${ENV_PATH}/dipu_env
            bash scripts/increment_coverage.sh ${REQUIRE_COVERAGE}
          else
            ssh SH1424 """
            set -e
            cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/
            rm -rf scripts
            ln -s ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/dipu/third_party/DIOPI/scripts scripts
            source /mnt/cache/share/platform/env/pt2.0_diopi
            bash scripts/increment_coverage.sh ${REQUIRE_COVERAGE}
            """
          fi

  Test-One-Iter_Cuda:
    name: Test-one-iter-cuda
    runs-on: ${{ inputs.runner }}
    env:
      GETRUNNER: ${{ inputs.runner }}
      DEEPLINK_PATH: ${{ inputs.deeplink_path }}
      ENV_PATH: ${{ inputs.env_path }}
      CUDA_PARTATION: "pat_dev"
    steps:
      - name: build some env
        run: |
          if [[ "${GETRUNNER}" == *sco* ]];then
            set -e
            export basic_path=${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/
            srun --job-name=${GITHUB_JOB} bash -c "cd ${basic_path} \
            && export PYTHONPATH=${basic_path}/mmlab_pack:${basic_path}/mmlab_pack/mmengine:${basic_path}/mmlab_pack/mmcv:$PYTHONPATH \
            && source ${ENV_PATH}/dipu_env && cd mmlab_pack \
            && bash ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_one_iter.sh build_cuda"
          else
            ssh SH1424 """
            set -ex
            cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && cd Build-Cuda/dipu
            source ${ENV_PATH}/dipu_env
            basic_path=${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
            export PYTHONPATH=\${basic_path}/mmengine:\$PYTHONPATH
            export PYTHONPATH=\${basic_path}/mmcv:\$PYTHONPATH
            export PYTHONPATH=\$(pwd):\$PYTHONPATH
            cd mmlab_pack
            srun --job-name=${GITHUB_RUN_NUMBER}_${GITHUB_JOB} --partition=${CUDA_PARTATION} --gres=gpu:1 --time=20 bash ../scripts/ci/ci_one_iter.sh build_cuda
            """
          fi
      - name: run-one-iter-for-tradition
        run: |
          if [[ "${GETRUNNER}" == *sco* ]];then
            set -e
            cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
            && source ${ENV_PATH}/dipu_env && cd mmlab_pack \
            && rm -rf one_iter_data \
            && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco  ${GITHUB_JOB} gpu sco && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
          else
            ssh SH1424 """
              cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && cd Build-Cuda/dipu
              source ${ENV_PATH}/dipu_env
              basic_path=${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
              source scripts/ci/ci_one_iter.sh export_pythonpath_cuda \${basic_path}
              export PYTHONPATH=\$(pwd):\$PYTHONPATH
              cd mmlab_pack
              rm -rf one_iter_data
              python ../scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} "gpu:1" \"${CUDA_PARTATION}\" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
            """
          fi
      - name: run-one-iter-for-llm
        run: |
          if [[ "${GETRUNNER}" == *sco* ]];then
            set -e
            cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/ \
            && source ${ENV_PATH}/dipu_env && cd mmlab_pack \
            && rm -rf one_iter_data \
            && python ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/scripts/ci/ci_run_one_iter.py sco ${GITHUB_JOB} gpu sco "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
          else
            ssh SH1424 """
            cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && cd Build-Cuda/dipu
            source ${ENV_PATH}/dipu_env
            basic_path=${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
            source scripts/ci/ci_one_iter.sh export_pythonpath_cuda \${basic_path}
            export PYTHONPATH=\$(pwd):\$PYTHONPATH
            cd mmlab_pack
            rm -rf one_iter_data
            python ../scripts/ci/ci_run_one_iter.py cuda  ${GITHUB_RUN_NUMBER}_${GITHUB_JOB} "gpu:1" \"${CUDA_PARTATION}\" "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1)
            """
          fi
      - name: Perform cleanup one iter data
        if: always()
        run: |
          if [[ "${GETRUNNER}" == *sco* ]];then
            set -e
            cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
            rm -rf one_iter_data
            touch one_iter_data   #用于占位，防止创建新的 one_iter_data 文件夹
          else
            ssh SH1424 """
            set -ex
            echo "${GITHUB_RUN_NUMBER}_${GITHUB_JOB}"
            scancel -n "${GITHUB_RUN_NUMBER}_${GITHUB_JOB}"
            cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda/dipu/mmlab_pack
            rm -rf one_iter_data
            touch one_iter_data # 用于占位，防止创建新的 one_iter_data 文件夹
            """
          fi
      - name: Check for failure
        if: ${{ failure() }}
        run: exit 1

  Build-Cuda-Latest-Target:
    name: Build-dipu-cuda-latest-target
    runs-on: ${{ inputs.runner }}
    env:
      GETRUNNER: ${{ inputs.runner }}
      DEEPLINK_PATH: ${{ inputs.deeplink_path }}
      ENV_PATH: ${{ inputs.env_path }}
      CUDA_PARTATION: "pat_dev"
    steps:
      - name: Build dipu diopi-latest-target
        run: |
          if [[ "${GETRUNNER}" == *sco* ]];then
            set -e
            cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source-main ${GITHUB_JOB}
            srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB}/dipu \
            && source ${ENV_PATH}/dipu_env  \
            && bash scripts/ci/nv/ci_nv_script.sh build_dipu" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
          else
            ssh SH1424 """
            set -ex
            cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source-main ${GITHUB_JOB}  && cd ${GITHUB_JOB}/dipu
            source ${ENV_PATH}/dipu_env
            srun --job-name=${GITHUB_RUN_NUMBER}_${GITHUB_JOB} --partition=${CUDA_PARTATION} --gres=gpu:1 --cpus-per-task=5 --mem=16G --time=30 bash scripts/ci/nv/ci_nv_script.sh build_dipu \
            || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
            """
          fi

  Test-Cuda-Latest-Target:
    name: Test-dipu-cuda-latest-target
    needs: [Build-Cuda-Latest-Target]
    runs-on: ${{ inputs.runner }}
    env:
      GETRUNNER: ${{ inputs.runner }}
      DEEPLINK_PATH: ${{ inputs.deeplink_path }}
      ENV_PATH: ${{ inputs.env_path }}
      CUDA_PARTATION: "pat_dev"
    steps:
      - name: Run-test
        run: |
          if [[ "${GETRUNNER}" == *sco* ]];then
            set -e
            srun --job-name=need_two_gpus bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda-Latest-Target/dipu \
            && source ${ENV_PATH}/dipu_env  \
            && bash tests/run_nv_tests.sh" && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Cuda-Latest-Target
          else
            ssh SH1424 """
            set -ex
            cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && cd Build-Cuda-Latest-Target/dipu
            source ${ENV_PATH}/dipu_env
            srun --job-name=${GITHUB_RUN_NUMBER}_${GITHUB_JOB} --partition=${CUDA_PARTATION} --gres=gpu:2 --cpus-per-task=5 --mem=16G --time=60 sh tests/run_nv_tests.sh && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Cuda-Latest-Target \
            || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Cuda-Latest-Target && exit 1 )
            """
          fi


  Build-Cuda-Pt211:
    name: Build-dipu-cuda-pt211
    runs-on: ${{ inputs.runner }}
    env:
      GETRUNNER: ${{ inputs.runner }}
      DEEPLINK_PATH: ${{ inputs.deeplink_path }}
      ENV_PATH: ${{ inputs.env_path }}
      CUDA_PARTATION: "pat_dev"
    steps:
      - name: Build dipu
        run: |
          if [[ "${GETRUNNER}" == *sco* ]];then
            set -e
            cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB}
            srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB}/dipu \
            && source ${ENV_PATH}/dipu_env 2.1.1 \
            && bash scripts/ci/nv/ci_nv_script.sh build_dipu " || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
          else
            ssh SH1424 """
              set -ex
              cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB} && cd ${GITHUB_JOB}/dipu
              source ${ENV_PATH}/dipu_env 2.1.1
              srun --job-name=${GITHUB_RUN_NUMBER}_${GITHUB_JOB} --partition=${CUDA_PARTATION} --gres=gpu:1 --cpus-per-task=5 --mem=16G --time=60 bash scripts/ci/nv/ci_nv_script.sh build_dipu \
              || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
              """
          fi

  Test-Cuda-Pt211:
    name: Test-dipu-cuda-pt211
    needs: [Build-Cuda-Pt211]
    runs-on: ${{ inputs.runner }}
    env:
      GETRUNNER: ${{ inputs.runner }}
      DEEPLINK_PATH: ${{ inputs.deeplink_path }}
      ENV_PATH: ${{ inputs.env_path }}
      CUDA_PARTATION: "pat_dev"
    steps:
      - name: Run-test
        run: |
          if [[ "${GETRUNNER}" == *sco* ]];then
            set -e
            srun --job-name=need_two_gpus bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda-Pt211/dipu \
            && source ${ENV_PATH}/dipu_env 2.1.1 \
            && bash tests/run_nv_tests.sh" && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Cuda-Pt211
          else
            ssh SH1424 """
            set -ex
            cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda-Pt211/dipu
            source ${ENV_PATH}/dipu_env 2.1.1
            srun --job-name=${GITHUB_RUN_NUMBER}_${GITHUB_JOB} --partition=${CUDA_PARTATION} --gres=gpu:2 --cpus-per-task=5 bash tests/run_nv_tests.sh \
            || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Cuda-Pt211 && exit 1 )
            """
          fi

  Build-Cuda-Pt222:
    name: Build-dipu-cuda-pt222
    runs-on: ${{ inputs.runner }}
    env:
      GETRUNNER: ${{ inputs.runner }}
      DEEPLINK_PATH: ${{ inputs.deeplink_path }}
      ENV_PATH: ${{ inputs.env_path }}
      CUDA_PARTATION: "pat_dev"
    steps:
      - name: Build dipu
        run: |
          if [[ "${GETRUNNER}" == *sco* ]];then
            set -e
            cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB}
            srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB}/dipu \
            && source ${ENV_PATH}/dipu_env 2.2.2 \
            && bash scripts/ci/nv/ci_nv_script.sh build_dipu " || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
          else
            ssh SH1424 """
              set -ex
              cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB} && cd ${GITHUB_JOB}/dipu
              source ${ENV_PATH}/dipu_env 2.2.2
              srun --job-name=${GITHUB_RUN_NUMBER}_${GITHUB_JOB} --partition=${CUDA_PARTATION} --gres=gpu:1 --cpus-per-task=5 --mem=16G --time=60 bash scripts/ci/nv/ci_nv_script.sh build_dipu \
              || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
              """
          fi

  Test-Cuda-Pt222:
    name: Test-dipu-cuda-pt222
    needs: [Build-Cuda-Pt222]
    runs-on: ${{ inputs.runner }}
    env:
      GETRUNNER: ${{ inputs.runner }}
      DEEPLINK_PATH: ${{ inputs.deeplink_path }}
      ENV_PATH: ${{ inputs.env_path }}
      CUDA_PARTATION: "pat_dev"
    steps:
      - name: Run-test
        run: |
          if [[ "${GETRUNNER}" == *sco* ]];then
            set -e
            srun --job-name=need_two_gpus bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda-Pt222/dipu \
            && source ${ENV_PATH}/dipu_env 2.2.2 \
            && bash tests/run_nv_tests.sh" && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Cuda-Pt222
          else
            ssh SH1424 """
            set -ex
            cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Cuda-Pt222/dipu
            source ${ENV_PATH}/dipu_env 2.2.2
            srun --job-name=${GITHUB_RUN_NUMBER}_${GITHUB_JOB} --partition=${CUDA_PARTATION} --gres=gpu:2 --cpus-per-task=5 bash tests/run_nv_tests.sh \
            || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Cuda-Pt222 && exit 1 )
            """
          fi