feat: implement evaluation framework for praisonaiagents

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+patreon: MervinPraison
+ko_fi: MervinPraison

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,34 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: '[BUG] '
+labels: bug
+assignees: ''
+---
+
+## Environment
+- Provider (select one):
+  - [ ] Anthropic
+  - [ ] OpenAI
+  - [ ] Google Vertex AI
+  - [ ] AWS Bedrock
+  - [ ] Other: <!-- specify -->
+- PraisonAI version: <!-- if known -->
+- Operating System: <!-- e.g. macOS 14.3, Windows 11, Ubuntu 22.04 -->
+
+## Full Code
+<!-- Full code that reproduces the bug -->
+
+## Steps to Reproduce
+1. <!-- First step -->
+2. <!-- Second step -->
+3. <!-- And so on... -->
+
+## Expected Behavior
+<!-- What you expected to happen -->
+
+## Actual Behavior
+<!-- What actually happened -->
+
+## Additional Context
+<!-- Add any other context about the problem here, such as screenshots, logs, etc. --> 

.github/actions/claude-code-action/action.yml

@@ -0,0 +1,240 @@
+name: "Claude Code Action"
+description: "Run Claude Code in GitHub Actions workflows"
+
+inputs:
+  anthropic_api_key:
+    description: "Anthropic API key"
+    required: false
+  github_token:
+    description: "GitHub token for Claude to operate with"
+    required: false
+    default: ${{ github.token }}
+  trigger_phrase:
+    description: "The trigger phrase to look for in comments, issue/PR bodies, and issue titles"
+    required: false
+    default: "@claude"
+  assignee_trigger:
+    description: "The assignee username that triggers the action (e.g. @claude). Only used for issue assignment"
+    required: false
+  max_turns:
+    description: "Maximum number of conversation turns Claude can take"
+    required: false
+  timeout_minutes:
+    description: "Timeout in minutes for execution"
+    required: false
+    default: "30"
+  model:
+    description: "Model to use (provider-specific format required for Bedrock/Vertex)"
+    required: false
+  use_bedrock:
+    description: "Use Amazon Bedrock with OIDC authentication instead of direct Anthropic API"
+    required: false
+    default: "false"
+  use_vertex:
+    description: "Use Google Vertex AI with OIDC authentication instead of direct Anthropic API"
+    required: false
+    default: "false"
+  allowed_tools:
+    description: "Additional tools for Claude to use (the base GitHub tools will always be included)"
+    required: false
+    default: ""
+  disallowed_tools:
+    description: "Tools that Claude should never use"
+    required: false
+    default: ""
+  custom_instructions:
+    description: "Additional custom instructions to include in the prompt for Claude"
+    required: false
+    default: ""
+  mcp_config:
+    description: "Additional MCP configuration (JSON string) that merges with the built-in GitHub MCP servers"
+    required: false
+    default: ""
+  claude_env:
+    description: "Custom environment variables to pass to Claude Code execution (YAML format)"
+    required: false
+    default: ""
+  direct_prompt:
+    description: "Direct prompt for Claude to execute automatically without needing a trigger (for automated workflows)"
+    required: false
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install Claude Code
+      shell: bash
+      run: npm install -g @anthropic-ai/claude-code
+
+    - name: Install GitHub MCP Server
+      shell: bash
+      run: |
+        claude mcp add-json github '{
+          "command": "docker",
+          "args": [
+            "run",
+            "-i",
+            "--rm",
+            "-e",
+            "GITHUB_PERSONAL_ACCESS_TOKEN",
+            "ghcr.io/github/github-mcp-server:sha-ff3036d"
+          ],
+          "env": {
+            "GITHUB_PERSONAL_ACCESS_TOKEN": "${{ inputs.github_token }}"
+          }
+        }'
+
+    - name: Extract GitHub Context and Create Prompt
+      shell: bash
+      id: prepare_context
+      run: |
+        echo "🔍 Extracting GitHub context from event: ${{ github.event_name }}"
+
+        # Function to check for trigger phrase
+        check_trigger() {
+          local text="$1"
+          local trigger="${{ inputs.trigger_phrase }}"
+          if [[ "$text" == *"$trigger"* ]]; then
+            return 0
+          fi
+          return 1
+        }
+
+        # Extract context based on event type
+        TRIGGER_FOUND="false"
+        USER_REQUEST=""
+        CONTEXT_INFO=""
+
+        case "${{ github.event_name }}" in
+          "issue_comment")
+            COMMENT_BODY="${{ github.event.comment.body }}"
+            ISSUE_TITLE="${{ github.event.issue.title }}"
+            ISSUE_NUMBER="${{ github.event.issue.number }}"
+
+            if check_trigger "$COMMENT_BODY"; then
+              TRIGGER_FOUND="true"
+              USER_REQUEST="$COMMENT_BODY"
+              CONTEXT_INFO="Issue Comment on #$ISSUE_NUMBER: $ISSUE_TITLE"
+            fi
+            ;;
+
+          "pull_request_review_comment")
+            COMMENT_BODY="${{ github.event.comment.body }}"
+            PR_TITLE="${{ github.event.pull_request.title }}"
+            PR_NUMBER="${{ github.event.pull_request.number }}"
+
+            if check_trigger "$COMMENT_BODY"; then
+              TRIGGER_FOUND="true"
+              USER_REQUEST="$COMMENT_BODY"
+              CONTEXT_INFO="PR Comment on #$PR_NUMBER: $PR_TITLE"
+            fi
+            ;;
+
+          "pull_request_review")
+            REVIEW_BODY="${{ github.event.review.body }}"
+            PR_TITLE="${{ github.event.pull_request.title }}"
+            PR_NUMBER="${{ github.event.pull_request.number }}"
+
+            if check_trigger "$REVIEW_BODY"; then
+              TRIGGER_FOUND="true"
+              USER_REQUEST="$REVIEW_BODY"
+              CONTEXT_INFO="PR Review on #$PR_NUMBER: $PR_TITLE"
+            fi
+            ;;
+
+          "issues")
+            ISSUE_BODY="${{ github.event.issue.body }}"
+            ISSUE_TITLE="${{ github.event.issue.title }}"
+            ISSUE_NUMBER="${{ github.event.issue.number }}"
+
+            if check_trigger "$ISSUE_TITLE" || check_trigger "$ISSUE_BODY"; then
+              TRIGGER_FOUND="true"
+              USER_REQUEST="$ISSUE_BODY"
+              CONTEXT_INFO="Issue #$ISSUE_NUMBER: $ISSUE_TITLE"
+            elif [[ "${{ github.event.action }}" == "assigned" && -n "${{ inputs.assignee_trigger }}" ]]; then
+              ASSIGNEE="${{ github.event.assignee.login }}"
+              if [[ "$ASSIGNEE" == "${{ inputs.assignee_trigger }}" ]]; then
+                TRIGGER_FOUND="true"
+                USER_REQUEST="$ISSUE_BODY"
+                CONTEXT_INFO="Issue #$ISSUE_NUMBER assigned to $ASSIGNEE: $ISSUE_TITLE"
+              fi
+            fi
+            ;;
+        esac
+
+        # Check for direct prompt override
+        if [[ -n "${{ inputs.direct_prompt }}" ]]; then
+          TRIGGER_FOUND="true"
+          USER_REQUEST="${{ inputs.direct_prompt }}"
+          CONTEXT_INFO="Automated GitHub workflow"
+        fi
+
+        if [[ "$TRIGGER_FOUND" != "true" ]]; then
+          echo "❌ No trigger phrase found or direct prompt provided. Exiting gracefully."
+          echo "SKIP_EXECUTION=true" >> $GITHUB_ENV
+          exit 0
+        fi
+
+        echo "✅ Trigger found! Context: $CONTEXT_INFO"
+
+        # Create comprehensive prompt
+        mkdir -p /tmp/claude-action
+        cat > /tmp/claude-action/github-context-prompt.txt << EOF
+        You are Claude Code, an AI assistant helping with GitHub workflows and code.
+
+        Repository: ${{ github.repository }}
+        Context: $CONTEXT_INFO
+        Event: ${{ github.event_name }}
+
+        User Request:
+        $USER_REQUEST
+
+        Please analyze the request and provide helpful assistance. You have access to repository tools and can help with:
+        - Code analysis and implementation
+        - GitHub workflows and automation
+        - Pull request reviews and feedback
+        - Issue resolution and bug fixes
+        - Documentation updates
+        - Testing and deployment
+
+        Respond naturally and helpfully to the user's request using the available tools.
+        EOF
+
+        echo "PROMPT_FILE=/tmp/claude-action/github-context-prompt.txt" >> $GITHUB_ENV
+        echo "SKIP_EXECUTION=false" >> $GITHUB_ENV
+
+    - name: Run Claude Code
+      if: env.SKIP_EXECUTION != 'true'
+      shell: bash
+      run: |
+        echo "🚀 Running Claude Code with GitHub context..."
+
+        # Build command arguments
+        CMD_ARGS=("-p" "--verbose" "--output-format" "stream-json")
+
+        # Add max turns if specified
+        if [[ -n "${{ inputs.max_turns }}" ]]; then
+          CMD_ARGS+=("--max-turns" "${{ inputs.max_turns }}")
+        fi
+
+        # Add allowed tools (include GitHub tools by default)
+        TOOLS="mcp__github__get_issue,mcp__github__get_issue_comments,mcp__github__update_issue,mcp__github__search_issues,mcp__github__list_issues,mcp__github__create_comment,Read,Write,Edit,Bash"
+        if [[ -n "${{ inputs.allowed_tools }}" ]]; then
+          TOOLS="$TOOLS,${{ inputs.allowed_tools }}"
+        fi
+        CMD_ARGS+=("--allowedTools" "$TOOLS")
+
+        # Add disallowed tools
+        if [[ -n "${{ inputs.disallowed_tools }}" ]]; then
+          CMD_ARGS+=("--disallowedTools" "${{ inputs.disallowed_tools }}")
+        fi
+
+        echo "📝 Executing Claude Code with prompt from file..."
+
+        # Execute Claude Code with timeout, using stdin for the prompt
+        TIMEOUT_SECONDS=$((${{ inputs.timeout_minutes }} * 60))
+        timeout $TIMEOUT_SECONDS claude "${CMD_ARGS[@]}" < "${{ env.PROMPT_FILE }}"
+
+        echo "✅ Claude Code execution completed"
+      env:
+        ANTHROPIC_API_KEY: ${{ inputs.anthropic_api_key }}
+        GITHUB_TOKEN: ${{ inputs.github_token }} 

.github/actions/claude-issue-triage-action/action.yml

@@ -0,0 +1,87 @@
+name: "Claude Issue Triage Action"
+description: "Automatically triage GitHub issues using Claude Code"
+
+inputs:
+  timeout_minutes:
+    description: "Timeout in minutes for execution"
+    required: false
+    default: "5"
+  anthropic_api_key:
+    description: "Anthropic API key"
+    required: true
+  github_token:
+    description: "GitHub token with repo and issues permissions"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout repository code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Create prompt file
+      shell: bash
+      run: |
+        mkdir -p /tmp/claude-prompts
+        cat > /tmp/claude-prompts/claude-issue-triage-prompt.txt << 'EOF'
+        You're an issue triage assistant for GitHub issues. Your task is to analyze the issue and select appropriate labels from the provided list.
+
+        IMPORTANT: Don't post any comments or messages to the issue. Your only action should be to apply labels.
+
+        Issue Information:
+        - REPO: ${{ github.repository }}
+        - ISSUE_NUMBER: ${{ github.event.issue.number }}
+
+        TASK OVERVIEW:
+
+        1. First, fetch the list of labels available in this repository by running: `gh label list`. Run exactly this command with nothing else.
+
+        2. Next, use the GitHub tools to get context about the issue:
+           - You have access to these tools:
+             - mcp__github__get_issue: Use this to retrieve the current issue's details including title, description, and existing labels
+             - mcp__github__get_issue_comments: Use this to read any discussion or additional context provided in the comments
+             - mcp__github__update_issue: Use this to apply labels to the issue (do not use this for commenting)
+             - mcp__github__search_issues: Use this to find similar issues that might provide context for proper categorization and to identify potential duplicate issues
+             - mcp__github__list_issues: Use this to understand patterns in how other issues are labeled
+           - Start by using mcp__github__get_issue to get the issue details
+
+        3. Analyze the issue content, considering:
+           - The issue title and description
+           - The type of issue (bug report, feature request, question, etc.)
+           - Technical areas mentioned
+           - Severity or priority indicators
+           - User impact
+           - Components affected
+
+        4. Select appropriate labels from the available labels list provided above:
+           - Choose labels that accurately reflect the issue's nature
+           - Be specific but comprehensive
+           - Select priority labels if you can determine urgency (high-priority, med-priority, or low-priority)
+           - Consider platform labels (android, ios) if applicable
+           - If you find similar issues using mcp__github__search_issues, consider using a "duplicate" label if appropriate. Only do so if the issue is a duplicate of another OPEN issue.
+
+        5. Apply the selected labels:
+           - Use mcp__github__update_issue to apply your selected labels
+           - DO NOT post any comments explaining your decision
+           - DO NOT communicate directly with users
+           - If no labels are clearly applicable, do not apply any labels
+
+        IMPORTANT GUIDELINES:
+        - Be thorough in your analysis
+        - Only select labels from the provided list above
+        - DO NOT post any comments to the issue
+        - Your ONLY action should be to apply labels using mcp__github__update_issue
+        - It's okay to not add any labels if none are clearly applicable
+        EOF
+
+    - name: Run Claude Code
+      uses: ./.github/actions/claude-code-action
+      with:
+        prompt_file: /tmp/claude-prompts/claude-issue-triage-prompt.txt
+        allowed_tools: "Bash(gh label list),mcp__github__get_issue,mcp__github__get_issue_comments,mcp__github__update_issue,mcp__github__search_issues,mcp__github__list_issues"
+        install_github_mcp: "true"
+        timeout_minutes: ${{ inputs.timeout_minutes }}
+        anthropic_api_key: ${{ inputs.anthropic_api_key }}
+        github_token: ${{ inputs.github_token }}