diff --git a/.github/actions/get-pr-details/action.yml b/.github/actions/get-pr-details/action.yml new file mode 100644 index 000000000..4d866651c --- /dev/null +++ b/.github/actions/get-pr-details/action.yml @@ -0,0 +1,134 @@ +name: Get PR Details +description: 'Retrieves pull request details for pull_request_target or merge_group events' + +inputs: + event_name: + description: 'Event name that triggered the caller workflow' + required: true + pr_number: + description: 'PR number for pull_request_target events' + required: false + merge_group_head_ref: + description: 'Merge group head ref for merge_group events' + required: false + +outputs: + pr_number: + description: 'Pull request number' + value: ${{ steps.get_pr_details.outputs.pr_number }} + pr_title: + description: 'Pull request title' + value: ${{ steps.get_pr_details.outputs.pr_title }} + pr_labels: + description: 'Pull request labels (comma-separated)' + value: ${{ steps.get_pr_details.outputs.pr_labels }} + pr_base_ref: + description: 'Pull request base reference' + value: ${{ steps.get_pr_details.outputs.pr_base_ref }} + skip_gaudi_tests: + description: 'Whether to skip Gaudi tests' + value: ${{ steps.get_pr_details.outputs.skip_gaudi_tests }} + +runs: + using: 'composite' + steps: + - name: Retrieve Pull Request information + id: get_pr_details + uses: actions/github-script@v7 + env: + EVENT_NAME: ${{ inputs.event_name }} + PR_NUMBER: ${{ inputs.pr_number }} + MERGE_GROUP_HEAD_REF: ${{ inputs.merge_group_head_ref }} + with: + script: | + const eventName = process.env.EVENT_NAME; + + console.log('Trigger event: %s', eventName); + if (eventName === 'merge_group') { + const headRef = process.env.MERGE_GROUP_HEAD_REF; + + if (!headRef || typeof headRef !== 'string') { + console.log('āŒ MERGE_GROUP_HEAD_REF is missing or invalid. Received:', headRef); + core.setFailed('MERGE_GROUP_HEAD_REF is missing or invalid for merge_group event'); + return; + } + + const prMatch = headRef.match(/pr-(\d+)-/); + + if (!prMatch) { + console.log('āŒ Merge group head ref does not match expected format "pr--...". Received:', headRef); + core.setFailed('Could not extract PR number from merge group head ref: unexpected format'); + return; + } + + const prNumber = parseInt(prMatch[1], 10); + console.log('šŸ“‹ Found source PR number: %s', prNumber); + + try { + const { data: pullRequest } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber + }); + + const prLabels = pullRequest.labels.map(l => l.name).join(', ') || 'None'; + + console.log('\nāœ… Source Pull Request Information:'); + console.log('- PR Number: #%s', pullRequest.number); + console.log('- PR Title: %s', pullRequest.title); + console.log('- PR Author: %s', pullRequest.user.login); + console.log('- PR State: %s', pullRequest.state); + console.log('- PR Labels: %s', prLabels); + + const skipGaudiTests = pullRequest.labels.some(l => l.name === 'skip-gaudi-tests'); + + const fs = require('fs'); + const outputFile = process.env.GITHUB_OUTPUT; + fs.appendFileSync(outputFile, `pr_number=${pullRequest.number}\n`); + fs.appendFileSync(outputFile, `pr_title=${pullRequest.title}\n`); + fs.appendFileSync(outputFile, `pr_labels=${prLabels}\n`); + fs.appendFileSync(outputFile, `pr_base_ref=${pullRequest.base.ref}\n`); + fs.appendFileSync(outputFile, `skip_gaudi_tests=${skipGaudiTests}\n`); + + } catch (error) { + console.error('āŒ Error fetching PR details:', error.message); + core.setFailed(error.message); + } + } else if (eventName === 'pull_request_target') { + const prNumber = process.env.PR_NUMBER; + + try { + const { data: pullRequest } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber + }); + + const prLabels = pullRequest.labels.map(l => l.name).join(', ') || 'None'; + + console.log('\nāœ… Pull Request Information:'); + console.log('- PR Number: #%s', pullRequest.number); + console.log('- PR Title: %s', pullRequest.title); + console.log('- PR Author: %s', pullRequest.user.login); + console.log('- PR State: %s', pullRequest.state); + console.log('- PR Labels: %s', prLabels); + + const skipGaudiTests = pullRequest.labels.some(l => l.name === 'skip-gaudi-tests'); + + const fs = require('fs'); + const outputFile = process.env.GITHUB_OUTPUT; + fs.appendFileSync(outputFile, `pr_number=${pullRequest.number}\n`); + fs.appendFileSync(outputFile, `pr_title=${pullRequest.title}\n`); + fs.appendFileSync(outputFile, `pr_labels=${prLabels}\n`); + fs.appendFileSync(outputFile, `pr_base_ref=${pullRequest.base.ref}\n`); + fs.appendFileSync(outputFile, `skip_gaudi_tests=${skipGaudiTests}\n`); + + } catch (error) { + console.error('āŒ Error fetching PR details:', error.message); + core.setFailed(error.message); + } + } else { + const errorMessage = "Event not supported. Use 'pull_request_target' or 'merge_group'."; + console.error(errorMessage); + core.setFailed(errorMessage); + } diff --git a/tests/calibration_tests/ci_calibration_smoke_tests.sh b/tests/calibration_tests/ci_calibration_smoke_tests.sh new file mode 100644 index 000000000..43d59a61c --- /dev/null +++ b/tests/calibration_tests/ci_calibration_smoke_tests.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +# Exit immediately if a command exits with a non-zero status. +# This ensures that if any test fails, the script will stop. +set -e + +# --- Configuration --- +# Defines the path to the vllm-gaudi directory. +VLLM_GAUDI_PREFIX=${VLLM_GAUDI_PREFIX:-"vllm-gaudi"} +echo "VLLM_GAUDI_PREFIX: $VLLM_GAUDI_PREFIX" + +# Calibration output directory (cleaned up after each test) +CALIBRATION_OUTPUT_DIR="${VLLM_GAUDI_PREFIX}/tests/calibration_tests/tmp-calibration-output" + +# Dataset for calibration (using NeelNanda/pile-10k which auto-downloads) +CALIBRATION_DATASET="NeelNanda/pile-10k" + +# Minimal batch size and limit for smoke tests (we only verify the procedure works) +BATCH_SIZE=1 +LIMIT=1 + +cleanup_calibration_output() { + if [ -d "${CALIBRATION_OUTPUT_DIR}" ]; then + echo "Cleaning up calibration output directory..." + rm -rf "${CALIBRATION_OUTPUT_DIR}" + fi +} + +# Simple smoke calibration test using granite model +run_granite_calibration_test() { + echo "āž”ļø Testing calibration procedure on ibm-granite/granite-3.3-2b-instruct..." + cleanup_calibration_output + + PT_HPU_LAZY_MODE=1 "${VLLM_GAUDI_PREFIX}/calibration/calibrate_model.sh" \ + -m ibm-granite/granite-3.3-2b-instruct \ + -d "${CALIBRATION_DATASET}" \ + -o "${CALIBRATION_OUTPUT_DIR}" \ + -b ${BATCH_SIZE} \ + -l ${LIMIT} \ + -t 1 + + if [ $? -ne 0 ]; then + echo "Error: Calibration failed for ibm-granite/granite-3.3-2b-instruct" >&2 + exit 1 + fi + echo "āœ… Calibration for ibm-granite/granite-3.3-2b-instruct passed." + cleanup_calibration_output +} + +# Simple smoke calibration test using Qwen-2.5 model +run_qwen_calibration_test() { + echo "āž”ļø Testing calibration procedure on Qwen/Qwen2.5-0.5B-Instruct..." + cleanup_calibration_output + + PT_HPU_LAZY_MODE=1 "${VLLM_GAUDI_PREFIX}/calibration/calibrate_model.sh" \ + -m Qwen/Qwen2.5-0.5B-Instruct \ + -d "${CALIBRATION_DATASET}" \ + -o "${CALIBRATION_OUTPUT_DIR}" \ + -b ${BATCH_SIZE} \ + -l ${LIMIT} \ + -t 1 + + if [ $? -ne 0 ]; then + echo "Error: Calibration failed for Qwen/Qwen2.5-0.5B-Instruct" >&2 + exit 1 + fi + echo "āœ… Calibration for Qwen/Qwen2.5-0.5B-Instruct passed." + cleanup_calibration_output +} + +# Simple smoke test for vision language models calibration using Qwen-2.5-VL +# (afierka) Temporarily disabled due to some issues, will re-enable once fixed. [GAUDISW-246468] +# run_qwen_vl_calibration_test() { +# echo "āž”ļø Testing VLM calibration procedure on Qwen/Qwen2.5-VL-3B-Instruct..." +# cleanup_calibration_output + +# PT_HPU_LAZY_MODE=1 "${VLLM_GAUDI_PREFIX}/calibration/vlm-calibration/calibrate_model.sh" \ +# -m Qwen/Qwen2.5-VL-3B-Instruct \ +# -o "${CALIBRATION_OUTPUT_DIR}" \ +# -b ${BATCH_SIZE} \ +# -l ${LIMIT} \ +# -t 1 + +# if [ $? -ne 0 ]; then +# echo "Error: VLM Calibration failed for Qwen/Qwen2.5-VL-3B-Instruct" >&2 +# exit 1 +# fi +# echo "āœ… VLM Calibration for Qwen/Qwen2.5-VL-3B-Instruct passed." +# cleanup_calibration_output +# } + +# --- Utility Functions --- + +# Function to run all tests sequentially +launch_all_tests() { + echo "šŸš€ Starting all calibration test suites..." + run_granite_calibration_test + run_qwen_calibration_test + # run_qwen_vl_calibration_test # (afierka) Temporarily disabled due to some issues, will re-enable once fixed. [GAUDISW-246468] + echo "šŸŽ‰ All calibration test suites passed successfully!" +} + +# A simple usage function to guide the user +usage() { + echo "Usage: $0 [function_name]" + echo "If no function_name is provided, all tests will be run." + echo "" + echo "Available functions:" + declare -F | awk '{print " - " $3}' | grep --color=never "run_" +} + +# --- Script Entry Point --- + +# Default to 'launch_all_tests' if no function name is provided as an argument. +FUNCTION_TO_RUN=${1:-launch_all_tests} + +# Check if the provided argument corresponds to a declared function in this script. +if declare -f "$FUNCTION_TO_RUN" > /dev/null +then + "$FUNCTION_TO_RUN" +else + echo "āŒ Error: Function '${FUNCTION_TO_RUN}' is not defined." + echo "" + usage + exit 1 +fi