benchmark_v2.yml

name: Benchmark v2 Framework

on:
  workflow_dispatch:

env:
  HF_HOME: /mnt/cache
  TRANSFORMERS_IS_CI: yes
  # For gated repositories, we still need to agree to share information on the Hub repo. page in order to get access.
  # This token is created under the bot `hf-transformers-bot`.
  HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }}

jobs:
  benchmark-v2:
    name: Benchmark v2
    runs-on: ${{ inputs.runner }}
    if: |
      (github.event_name == 'pull_request' && contains( github.event.pull_request.labels.*.name, 'run-benchmark')) ||
      (github.event_name == 'schedule')
    container:
      image: ${{ inputs.container_image }}
      options: ${{ inputs.container_options }}
    steps:
      - name: Get repo
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.commit_sha || github.sha }}

      - name: Install benchmark dependencies
        run: |
          python3 -m pip install -r benchmark_v2/requirements.txt

      - name: Reinstall transformers in edit mode
        run: |
          python3 -m pip uninstall -y transformers
          python3 -m pip install -e ".[torch]"

      - name: Show installed libraries and their versions
        run: |
          python3 -m pip list
          python3 -c "import torch; print(f'PyTorch version: {torch.__version__}')"
          python3 -c "import torch; print(f'CUDA available: {torch.cuda.is_available()}')"
          python3 -c "import torch; print(f'CUDA device count: {torch.cuda.device_count()}')" || true
          nvidia-smi || true

      - name: Run benchmark v2
        working-directory: benchmark_v2
        run: |
          echo "Running benchmarks"
          python3 run_benchmarks.py \
          --commit-id '${{ inputs.commit_sha || github.sha }}' \
          --run-id '${{ inputs.run_id }}' \
          --push-to-hub '${{ inputs.benchmark_repo_id}}' \
          --token '${{ secrets.TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN }}' \
          --log-level INFO
        env:
          HF_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }}