integration.yml

name: Integration tests

on:
  workflow_dispatch:
    inputs:
      serving-version:
        description: 'The released version of DJL Serving.'
        required: true
        default: ''
      tag-suffix:
        description: 'Run tests on the specific tags suffix i.e. arch-{suffix}'
        required: false
        type: string
        default: 'nightly'
      image-repo:
        description: 'The repository to fetch images from'
        required: false
        type: string
        default: '185921645874.dkr.ecr.us-east-1.amazonaws.com/djl-ci-temp'
  workflow_call:
    inputs:
      serving-version:
        description: 'The released version of DJL Serving.'
        required: false
        type: string
        default: ''
      tag-suffix:
        description: 'Run tests on the specific tags suffix i.e. arch-{suffix}'
        required: false
        type: string
        default: 'nightly'
      image-repo:
        description: 'The repository to fetch images from'
        required: false
        type: string
        default: '185921645874.dkr.ecr.us-east-1.amazonaws.com/djl-ci-temp'
    outputs:
      failure_cpu:
        value: ${{ jobs.test.outputs.failure_cpu || '0' }}
      failure_gpu:
        value: ${{ jobs.test.outputs.failure_gpu || '0' }}
      failure_aarch64:
        value: ${{ jobs.test.outputs.failure_aarch64 || '0' }}
      failure_lmi:
        value: ${{ jobs.test.outputs.failure_lmi || '0' }}
      failure_trtllm:
        value: ${{ jobs.test.outputs.failure_trtllm || '0' }}


permissions:
  id-token: write
  contents: read

jobs:
  create-runners:
    runs-on: [self-hosted, scheduler]
    steps:
      - name: Create new G6 instance
        id: create_g6_1
        run: |
          cd /home/ubuntu/djl_benchmark_script/scripts
          token=$( curl -X POST -H "Authorization: token ${{ secrets.ACTION_RUNNER_PERSONAL_TOKEN }}" \
          https://api.github.com/repos/deepjavalibrary/djl-serving/actions/runners/registration-token \
          --fail \
          | jq '.token' | tr -d '"' )
          ./start_instance.sh action_g6 $token djl-serving
      - name: Create new G6 instance
        id: create_g6_2
        run: |
          cd /home/ubuntu/djl_benchmark_script/scripts
          token=$( curl -X POST -H "Authorization: token ${{ secrets.ACTION_RUNNER_PERSONAL_TOKEN }}" \
          https://api.github.com/repos/deepjavalibrary/djl-serving/actions/runners/registration-token \
          --fail \
          | jq '.token' | tr -d '"' )
          ./start_instance.sh action_g6 $token djl-serving
      - name: Create new G6 instance
        id: create_g6_3
        run: |
          cd /home/ubuntu/djl_benchmark_script/scripts
          token=$( curl -X POST -H "Authorization: token ${{ secrets.ACTION_RUNNER_PERSONAL_TOKEN }}" \
          https://api.github.com/repos/deepjavalibrary/djl-serving/actions/runners/registration-token \
          --fail \
          | jq '.token' | tr -d '"' )
          ./start_instance.sh action_g6 $token djl-serving
      - name: Create new G6 instance
        id: create_g6_4
        run: |
          cd /home/ubuntu/djl_benchmark_script/scripts
          token=$( curl -X POST -H "Authorization: token ${{ secrets.ACTION_RUNNER_PERSONAL_TOKEN }}" \
          https://api.github.com/repos/deepjavalibrary/djl-serving/actions/runners/registration-token \
          --fail \
          | jq '.token' | tr -d '"' )
          ./start_instance.sh action_g6 $token djl-serving
      - name: Create new G6 instance
        id: create_g6_5
        run: |
          cd /home/ubuntu/djl_benchmark_script/scripts
          token=$( curl -X POST -H "Authorization: token ${{ secrets.ACTION_RUNNER_PERSONAL_TOKEN }}" \
          https://api.github.com/repos/deepjavalibrary/djl-serving/actions/runners/registration-token \
          --fail \
          | jq '.token' | tr -d '"' )
          ./start_instance.sh action_g6 $token djl-serving
      - name: Create new G6 instance
        id: create_g6_6
        run: |
          cd /home/ubuntu/djl_benchmark_script/scripts
          token=$( curl -X POST -H "Authorization: token ${{ secrets.ACTION_RUNNER_PERSONAL_TOKEN }}" \
          https://api.github.com/repos/deepjavalibrary/djl-serving/actions/runners/registration-token \
          --fail \
          | jq '.token' | tr -d '"' )
          ./start_instance.sh action_g6 $token djl-serving
      - name: Create new G6 instance
        id: create_g6_7
        run: |
          cd /home/ubuntu/djl_benchmark_script/scripts
          token=$( curl -X POST -H "Authorization: token ${{ secrets.ACTION_RUNNER_PERSONAL_TOKEN }}" \
          https://api.github.com/repos/deepjavalibrary/djl-serving/actions/runners/registration-token \
          --fail \
          | jq '.token' | tr -d '"' )
          ./start_instance.sh action_g6 $token djl-serving
      - name: Create new Graviton instance
        id: create_aarch64
        run: |
          cd /home/ubuntu/djl_benchmark_script/scripts
          token=$( curl -X POST -H "Authorization: token ${{ secrets.ACTION_RUNNER_PERSONAL_TOKEN }}" \
          https://api.github.com/repos/deepjavalibrary/djl-serving/actions/runners/registration-token \
          --fail \
          | jq '.token' | tr -d '"' )
          ./start_instance.sh action_graviton $token djl-serving

      - name: Create new CPU instance
        id: create_cpu
        run: |
          cd /home/ubuntu/djl_benchmark_script/scripts
          token=$( curl -X POST -H "Authorization: token ${{ secrets.ACTION_RUNNER_PERSONAL_TOKEN }}" \
          https://api.github.com/repos/deepjavalibrary/djl-serving/actions/runners/registration-token \
          --fail \
          | jq '.token' | tr -d '"' )
          ./start_instance.sh action_cpu $token djl-serving
      - name: Create new P4D instance
        id: create_p4d_1
        run: |
          cd /home/ubuntu/djl_benchmark_script/scripts
          token=$( curl -X POST -H "Authorization: token ${{ secrets.ACTION_RUNNER_PERSONAL_TOKEN }}" \
          https://api.github.com/repos/deepjavalibrary/djl-serving/actions/runners/registration-token \
          --fail \
          | jq '.token' | tr -d '"' )
          ./start_instance.sh action_lmic_p4d $token djl-serving
      # - name: Create new P4DE instance
      #   id: create_p4de_1
      #   run: |
      #     cd /home/ubuntu/djl_benchmark_script/scripts
      #     token=$( curl -X POST -H "Authorization: token ${{ secrets.ACTION_RUNNER_PERSONAL_TOKEN }}" \
      #     https://api.github.com/repos/deepjavalibrary/djl-serving/actions/runners/registration-token \
      #     --fail \
      #     | jq '.token' | tr -d '"' )
      #     ./start_instance.sh action_lmic_p4de $token djl-serving
    outputs:
      g6_instance_id_1: ${{ steps.create_g6_1.outputs.action_g6_instance_id }}
      g6_instance_id_2: ${{ steps.create_g6_2.outputs.action_g6_instance_id }}
      g6_instance_id_3: ${{ steps.create_g6_3.outputs.action_g6_instance_id }}
      g6_instance_id_4: ${{ steps.create_g6_4.outputs.action_g6_instance_id }}
      g6_instance_id_5: ${{ steps.create_g6_5.outputs.action_g6_instance_id }}
      g6_instance_id_6: ${{ steps.create_g6_6.outputs.action_g6_instance_id }}
      g6_instance_id_7: ${{ steps.create_g6_7.outputs.action_g6_instance_id }}
      aarch64_instance_id: ${{ steps.create_aarch64.outputs.action_graviton_instance_id }}
      cpu_instance_id: ${{ steps.create_cpu.outputs.action_cpu_instance_id }}
      p4d_instance_id_1: ${{ steps.create_p4d_1.outputs.action_lmic_p4d_instance_id }}
      # p4de_instance_id_1: ${{ steps.create_p4de_1.outputs.action_lmic_p4de_instance_id }}

  test:
    runs-on:
      - 'self-hosted'
      - ${{ format('RUN_ID-{0}', github.run_id) }}
      - ${{ format('RUN_NUMBER-{0}', github.run_number) }}
      - ${{ format('SHA-{0}', github.sha) }}
      - ${{ format('JOB-{0}', 'create-runners') }}
      - ${{ matrix.test.instance }}
    timeout-minutes: 180
    needs: create-runners
    strategy:
      fail-fast: false
      matrix:
        test:
          - test: TestCpuFull
            instance: cpu
            failure-prefix: cpu
          - test: TestCpuBoth
            instance: cpu
            failure-prefix: cpu
          - test: TestGpu_g6
            instance: g6
            failure-prefix: gpu
          # - test: TestAarch64
          #   instance: aarch64
          #   failure-prefix: aarch64
          # - test: TestHfHandler_g6
          #   instance: g6
          #   failure-prefix: lmi
          # - test: TestTrtLlmHandler1_g6
          #   instance: g6
          #   failure-prefix: trtllm
          # - test: TestTrtLlmHandler2_g6
          #   instance: g6
          #   failure-prefix: trtllm
          - test: TestVllm1_g6
            instance: g6
            failure-prefix: lmi
          - test: TestVllm2_g6
            instance: g6
            failure-prefix: lmi
          - test: TestVllmCustomHandlers_g6
            instance: g6
            failure-prefix: lmi
          - test: TestVllmCustomFormatters_g6
            instance: g6
            failure-prefix: lmi
          - test: TestVllmLora_g6
            instance: g6
            failure-prefix: lmi
          - test: TestVllmAsyncLora_g6
            instance: g6
            failure-prefix: lmi
          - test: TestVllmAsyncLoraWithCustomCode_g6
            instance: g6
            failure-prefix: lmi
          - test: TestMultiModalVllm_p4d
            instance: p4d
            failure-prefix: lmi
          # - test: TestTextEmbedding_g6
          #   instance: g6
          #   failure-prefix: lmi
          # - test: TestCorrectnessTrtLlm_g6
          #   instance: g6
          #   failure-prefix: trtllm
          - test: TestStatefulModel_g6
            instance: g6
            failure-prefix: lmi
          - test: TestVllmLmcache_g6
            instance: g6
            failure-prefix: lmi
          - test: TestVllmLmcachePerformance_g6
            instance: g6
            failure-prefix: lmi
          # P4D instance tests
          - test: TestVllm_p4d
            instance: p4d
            failure-prefix: lmi
          # P4DE instance tests
          # - test: TestVllm_p4de
          #   instance: p4de
          #   failure-prefix: lmi
    outputs:
      failure_cpu: ${{ steps.test-failure.outputs.failure_cpu }}
      failure_gpu: ${{ steps.test-failure.outputs.failure_gpu }}
      failure_aarch64: ${{ steps.test-failure.outputs.failure_aarch64 }}
      failure_lmi: ${{ steps.test-failure.outputs.failure_lmi }}
      failure_trtllm: ${{ steps.test-failure.outputs.failure_trtllm }}

    steps:
      - uses: actions/checkout@v4
      - name: Clean env
        run: |
          yes | docker system prune -a --volumes
          sudo rm -rf /home/ubuntu/actions-runner/_work/_tool/Java_Corretto_jdk/
          echo "wait dpkg lock..."
          while sudo fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; do sleep 5; done
      - name: Set up JDK 17
        uses: actions/setup-java@v4
        with:
          distribution: 'corretto'
          java-version: 17
      - name: Set up Python3
        uses: actions/setup-python@v5
        with:
          python-version: '3.10.x'
      - name: Install pip dependencies
        run: |
          python -m pip install --upgrade pip
          pip install pytest requests "numpy<2" pillow huggingface_hub awscli torch
      - name: Install awscurl
        working-directory: tests/integration
        run: |
          wget https://publish.djl.ai/awscurl/awscurl
          chmod +x awscurl
          mkdir outputs
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::185921645874:role/github-actions-djl-serving
          aws-region: us-east-1
          role-duration-seconds: 7200
      - name: Test
        working-directory: tests/integration
        env:
          TEST_SERVING_VERSION: ${{ inputs.serving-version }}
          IMAGE_TAG_SUFFIX: ${{ inputs.tag-suffix }}
          IMAGE_REPO: ${{ inputs.image-repo }}
        run: |
          ECR_REGION=$(echo "$IMAGE_REPO" | awk -F. '{print $4}')
          if [[ -n "$ECR_REGION" ]]; then
            aws ecr get-login-password --region $ECR_REGION | docker login --username AWS --password-stdin "$IMAGE_REPO"
          fi
          python -m pytest -s -k ${{ matrix.test.test }} tests.py
      - name: Cleanup
        working-directory: tests/integration
        run: |
          rm -rf outputs
          rm awscurl
      - name: On Failure
        id: test-failure
        if: ${{ failure() }}
        working-directory: tests/integration
        run: |
          for file in outputs/*; do if [ -f "$file" ]; then echo "Contents of $file:"; cat "$file"; echo; fi; done
          sudo rm -rf outputs && sudo rm -rf models
          rm awscurl
          ./remove_container.sh
          failure_prefix="${{ matrix.test.failure-prefix }}"
          echo "failure_${failure_prefix}=1" >> "$GITHUB_OUTPUT"
      - name: Upload test logs
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: test-${{ matrix.test.test }}-logs
          path: tests/integration/all_logs/



  stop-runners:
    if: always()
    runs-on: [ self-hosted, scheduler ]
    needs: [ create-runners, test ]
    steps:
      - name: Stop all instances
        run: |
          cd /home/ubuntu/djl_benchmark_script/scripts
          instance_id=${{ needs.create-runners.outputs.g6_instance_id_1 }}
          ./stop_instance.sh $instance_id
          instance_id=${{ needs.create-runners.outputs.g6_instance_id_2 }}
          ./stop_instance.sh $instance_id
          instance_id=${{ needs.create-runners.outputs.g6_instance_id_3 }}
          ./stop_instance.sh $instance_id
          instance_id=${{ needs.create-runners.outputs.g6_instance_id_4 }}
          ./stop_instance.sh $instance_id
          instance_id=${{ needs.create-runners.outputs.g6_instance_id_5 }}
          ./stop_instance.sh $instance_id
          instance_id=${{ needs.create-runners.outputs.g6_instance_id_6 }}
          ./stop_instance.sh $instance_id
          instance_id=${{ needs.create-runners.outputs.g6_instance_id_7 }}
          ./stop_instance.sh $instance_id
          instance_id=${{ needs.create-runners.outputs.aarch64_instance_id }}
          ./stop_instance.sh $instance_id
          instance_id=${{ needs.create-runners.outputs.cpu_instance_id }}
          ./stop_instance.sh $instance_id
          instance_id=${{ needs.create-runners.outputs.p4d_instance_id_1 }}
          ./stop_instance.sh $instance_id
          # instance_id=${{ needs.create-runners.outputs.p4de_instance_id_1 }}
          # ./stop_instance.sh $instance_id