system-integration.yml

name: System Integration Tests

on:
  # Daily schedule - 3 AM UTC
  schedule:
    - cron: "0 3 * * 0"
  # Pull requests with 'full-integration' label
  pull_request:
    types: [labeled, synchronize]
  # Manual trigger
  workflow_dispatch:
    inputs:
      trigger_task_data:
        description: "competition-api webhook/trigger_task data (JSON)"
        required: true
        default: '{"challenge_repo_url": "git@github.com:tob-challenges/example-libpng.git", "challenge_repo_base_ref": "0cc367aaeaac3f888f255cee5d394968996f736e", "challenge_repo_head_ref": "fdacd5a1dcff42175117d674b0fda9f8a005ae88", "fuzz_tooling_url": "git@github.com:tob-challenges/oss-fuzz-aixcc.git", "fuzz_tooling_ref": "d5fbd68fca66e6fa4f05899170d24e572b01853d", "fuzz_tooling_project_name": "libpng", "duration": 7200}'
      build_timeout:
        description: "Timeout for fuzzer build in minutes"
        required: false
        default: ""
      vuln_timeout:
        description: "Timeout for vuln submission in minutes"
        required: false
        default: ""
      patch_timeout:
        description: "Timeout for patch submission in minutes"
        required: false
        default: ""
      seed_gen_timeout:
        description: "Timeout for seed-gen submission in minutes"
        required: false
        default: ""
      bundle_timeout:
        description: "Timeout for bundle submission in minutes"
        required: false
        default: ""
      sarif_timeout:
        description: "Timeout for SARIF submission in minutes"
        required: false
        default: ""
      merger_timeout:
        description: "Timeout for merger bot in minutes"
        required: false
        default: ""

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  FUZZER_BUILD_TIMEOUT: ${{ inputs.build_timeout || 25 }}
  VULN_TIMEOUT: ${{ inputs.vuln_timeout || 25 }}
  PATCH_TIMEOUT: ${{ inputs.patch_timeout || 25 }}
  BUNDLE_TIMEOUT: ${{ inputs.bundle_timeout || 5 }}
  SARIF_TIMEOUT: ${{ inputs.sarif_timeout || 5 }}
  SEED_GEN_TIMEOUT: ${{ inputs.seed_gen_timeout || 25 }}
  MERGER_TIMEOUT: ${{ inputs.merger_timeout || 10 }}

permissions:
  contents: read

jobs:
  integration:
    # Only run on PRs if they have the 'full-integration' label
    if: |
      github.event_name == 'schedule' ||
      github.event_name != 'pull_request' ||
      contains(github.event.pull_request.labels.*.name, 'full-integration')
    runs-on: gha-ubuntu-8
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          persist-credentials: false
          submodules: true

      - name: Set BUTTERCUP_NAMESPACE for PRs
        if: github.event_name == 'pull_request'
        run: |
          pull_number=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH")
          export BUTTERCUP_NAMESPACE="pr-${pull_number}-${{ github.run_number }}"
          echo "BUTTERCUP_NAMESPACE=${BUTTERCUP_NAMESPACE}" >> "$GITHUB_ENV"

      - name: Set BUTTERCUP_NAMESPACE for branch
        if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
        run: |
          export BUTTERCUP_NAMESPACE="${GITHUB_REF_NAME/\//-}-${{ github.run_number }}"
          echo "BUTTERCUP_NAMESPACE=${BUTTERCUP_NAMESPACE}" >> "$GITHUB_ENV"

      - name: Configure env file for minikube
        env:
          BUTTERCUP_NS: ${{ env.BUTTERCUP_NAMESPACE }}
          OPENAI_KEY: ${{ secrets.OPENAI_API_KEY }}
          ANTHROPIC_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          GEMINI_KEY: ${{ secrets.GEMINI_API_KEY }}
          LF_HOST: ${{ secrets.LANGFUSE_HOST }}
          LF_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }}
          LF_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }}
          OTEL_EP: ${{ secrets.OTEL_ENDPOINT }}
          OTEL_TK: ${{ secrets.OTEL_TOKEN }}
        run: |
          cp ../.github/ci-env.template env
          # Remove GHCR_AUTH line and lines for values we'll append
          sed -i "/^GHCR_AUTH=/d" env
          sed -i "/^BUTTERCUP_NAMESPACE=/d" env
          sed -i "/^OPENAI_API_KEY=/d" env
          sed -i "/^ANTHROPIC_API_KEY=/d" env
          sed -i "/^GEMINI_API_KEY=/d" env
          sed -i "/^LANGFUSE_HOST=/d" env
          sed -i "/^LANGFUSE_PUBLIC_KEY=/d" env
          sed -i "/^LANGFUSE_SECRET_KEY=/d" env
          sed -i "/^OTEL_ENDPOINT=/d" env
          sed -i "/^OTEL_TOKEN=/d" env
          # Append values safely using echo (avoids sed injection with special chars)
          {
            echo "BUTTERCUP_NAMESPACE=${BUTTERCUP_NS}"
            echo "OPENAI_API_KEY=${OPENAI_KEY}"
            echo "ANTHROPIC_API_KEY=${ANTHROPIC_KEY}"
            echo "GEMINI_API_KEY=${GEMINI_KEY}"
            echo "LANGFUSE_HOST=${LF_HOST}"
            echo "LANGFUSE_PUBLIC_KEY=${LF_PUBLIC_KEY}"
            echo "LANGFUSE_SECRET_KEY=${LF_SECRET_KEY}"
            echo "OTEL_ENDPOINT=${OTEL_EP}"
            echo "OTEL_TOKEN=${OTEL_TK}"
          } >> "env"
        working-directory: deployment

      - name: Run CRS
        run: FORCE=true make deploy

      - name: Submit custom task to the CRS
        if: github.event_name == 'workflow_dispatch'
        run: |
          kubectl port-forward -n "$BUTTERCUP_NAMESPACE" service/buttercup-ui 31323:1323 &
          sleep 5
          ./orchestrator/scripts/custom_task_crs.sh "$DATA"
          sleep 5
        env:
          DATA: ${{ inputs.trigger_task_data }}

      - name: Submit example-libpng task to the CRS
        if: github.event_name != 'workflow_dispatch'
        run: |
          kubectl port-forward -n "$BUTTERCUP_NAMESPACE" service/buttercup-ui 31323:1323 &
          sleep 5
          ./orchestrator/scripts/task_crs.sh
          sleep 5

      - name: Wait for fuzzer build processing
        timeout-minutes: ${{ fromJSON(env.FUZZER_BUILD_TIMEOUT) }}
        run: |
          while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=scheduler --tail=-1 | grep "buttercup.orchestrator.scheduler.scheduler - INFO - .* Processing build output for type FUZZER"; do
            sleep 60
          done

      - name: Wait for vuln to be found
        timeout-minutes: ${{ fromJSON(env.VULN_TIMEOUT) }}
        run: |
          while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=scheduler --tail=-1 | grep "] POV submission response: pov_id=" ; do
            sleep 60
          done
      - name: Wait for vuln to enter the passed state in competition api
        timeout-minutes: ${{ fromJSON(env.VULN_TIMEOUT) }}
        run: |
          while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=scheduler --tail=-1 | grep "Updated POV status. New status PASSED" ; do
            sleep 60
          done

      - name: Wait for seed-gen to submit at least 1 seed
        timeout-minutes: ${{ fromJSON(env.SEED_GEN_TIMEOUT) }}
        run: |
          while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=seed-gen -c seed-gen --tail=-1 | grep "Copied [1-9][0-9]* files to corpus"; do
            sleep 60
          done

      - name: Wait for merger-bot to submit files to remote corpus and prune local corpus
        timeout-minutes: ${{ fromJSON(env.MERGER_TIMEOUT) }}
        run: |
          while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=merger-bot -c merger-bot --tail=-1  | grep "Synced [1-9][0-9]* files that add coverage to remote corpus"; do
            sleep 60
          done
          while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=merger-bot -c merger-bot --tail=-1  | grep "Removed [1-9][0-9]* files from local corpus .* that don't add coverage"; do
            sleep 60
          done

      - name: Wait for patch to be recorded
        timeout-minutes: ${{ fromJSON(env.PATCH_TIMEOUT) }}
        run: |
          while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=scheduler --tail=-1 | grep "Appending patch for task"; do
            sleep 60
          done
      - name: Approve the patch in the buttercup-ui
        run: |
          # Wait until a log line with competition_patch_id= is available, then extract PATCH_ID and TASK_ID from that line
          while true; do
            PATCH_LINE=$(kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=scheduler --tail=-1 | grep "competition_patch_id=" | head -n 1)
            if [ -n "$PATCH_LINE" ]; then
              break
            fi
            echo "Waiting for PATCH_ID to be available..."
            sleep 10
          done
          echo "Patch log line: $PATCH_LINE"
          # Extract PATCH_ID from the log line (after 'competition_patch_id=' and before any space or end of line)
          PATCH_ID=$(echo "$PATCH_LINE" | sed -n 's/.*competition_patch_id=\([^ ]*\).*/\1/p')
          # Extract TASK_ID from the log line (between the first '[' and the first ']'), then after the first ':' (if present)
          TASK_ID=$(echo "$PATCH_LINE" | sed -n 's/.*\[\([^]]*\)\].*/\1/p' | sed 's/^[^:]*://')
          echo "Patch ID: $PATCH_ID"
          echo "Task ID: $TASK_ID"
          kubectl port-forward -n "$BUTTERCUP_NAMESPACE" service/buttercup-ui 31323:1323 &
          sleep 5
          curl -X POST "http://localhost:31323/v1/task/${TASK_ID}/patch/${PATCH_ID}/approve"
          sleep 5
      - name: Wait for patch to enter the passed state in competition api
        timeout-minutes: ${{ fromJSON(env.PATCH_TIMEOUT) }}
        run: |
          while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=scheduler --tail=-1 | grep "Patch passed"; do
            sleep 60
          done

      - name: Wait for bundle to be submitted
        timeout-minutes: ${{ fromJSON(env.BUNDLE_TIMEOUT) }}
        run: |
          while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=scheduler --tail=-1 | grep "Bundle submission response: bundle_id="; do
            sleep 60
          done

      - name: Send a SARIF broadcast
        timeout-minutes: ${{ fromJSON(env.SARIF_TIMEOUT) }}
        run: |
          TASK_ID=$(kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=scheduler --tail=-1 | grep "Submitting bundle for harness" | grep -o "\[[^]]*\]" | grep -o "[^[]*$" | cut -d: -f3 | tr -d ']')
          echo "Task ID: $TASK_ID"
          while true; do
            # Kill any existing port-forward processes
            pkill -f "kubectl port-forward.*buttercup-competition-api" || true

            # Start port forwarding in background
            kubectl port-forward -n "$BUTTERCUP_NAMESPACE" service/buttercup-ui 31323:1323 &

            # Give port-forward time to establish
            sleep 5

            # Try to send SARIF report
            if ./orchestrator/scripts/send_sarif.sh "$TASK_ID"; then
              # Success - exit loop
              break
            fi

            # Failed - wait a bit before retrying
            sleep 10
          done
      - name: Wait for Bundle to be patched to include the SARIF
        timeout-minutes: ${{ fromJSON(env.SARIF_TIMEOUT) }}
        run: |
          while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=scheduler --tail=-1 | grep "Bundle patch submission response: broadcast_sarif_id="; do
            sleep 60
          done
      - name: Wait for SARIF to be submitted as correct
        timeout-minutes: ${{ fromJSON(env.SARIF_TIMEOUT) }}
        run: |
          while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=scheduler --tail=-1 | grep "Matching SARIF submission response"; do
            sleep 60
          done

      # Disable them for now because they can contain sensitive information
      # - name: Collect logs
      #   run: |
      #     ./collect-logs.sh
      #     mkdir -p docker_logs
      #     cp -rv crs_pod_logs_* docker_logs
      #   if: always()
      #   working-directory: deployment

      - name: Turn off CRS
        run: |
          make undeploy
          make clean-local
        if: always()

      # - name: Upload Docker logs
      #   uses: actions/upload-artifact@v4
      #   with:
      #     name: docker-logs
      #     path: deployment/docker_logs/
      #     retention-days: 4
      #   if: always()