AI Bug Detection in Community Discussions #54
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: AI Bug Detection in Community Discussions | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 16 * * *' # Run every day at 4:00 PM UTC | |
| permissions: | |
| models: read | |
| contents: read | |
| issues: write # Added permission to create issues | |
| jobs: | |
| fetch-and-analyze: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Install Playwright | |
| run: | | |
| npm init -y | |
| npm install playwright | |
| - name: Install Playwright browsers | |
| run: npx playwright install --with-deps | |
| - name: Fetch discussions | |
| run: node scripts/fetch-discussions.js | |
| - name: Analyze discussions with AI | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| if [ -f "discussions.json" ]; then | |
| echo "=== ANALYZING DISCUSSIONS WITH GPT-4O ===" | |
| # Install jq if not available | |
| if ! command -v jq &> /dev/null; then | |
| apt-get update && apt-get install -y jq | |
| fi | |
| # Read discussions data | |
| DISCUSSIONS=$(cat discussions.json) | |
| # Check if there are any discussions | |
| if [ "$(echo "$DISCUSSIONS" | jq 'length')" -eq 0 ]; then | |
| echo "No discussions found from the last 24 hours." | |
| exit 0 | |
| fi | |
| # Create a simplified discussion list for AI analysis | |
| DISCUSSION_LIST=$(echo "$DISCUSSIONS" | jq -r '.[] | "Title: \(.title) | Author: \(.author) | Comments: \(.commentCount) | Date: \(.timeText) | URL: \(.url)"') | |
| # Create AI analysis prompt | |
| cat > analysis_prompt.txt << 'EOF' | |
| You are an expert at analyzing GitHub discussions to identify potential bug reports and categorize discussions. | |
| Please analyze the following GitHub discussions from the Models category and classify each one into one of these categories: | |
| - BUG_REPORT: Issues, problems, errors, crashes, things not working as expected | |
| - FEATURE_REQUEST: Requests for new features, enhancements, improvements | |
| - QUESTION: How-to questions, help requests, clarifications, documentation questions | |
| - DISCUSSION: General discussions, announcements, sharing experiences | |
| For each discussion, provide: | |
| 1. Classification (BUG_REPORT, FEATURE_REQUEST, QUESTION, or DISCUSSION) | |
| 2. Confidence score (0-100%) | |
| 3. Brief reasoning (1-2 sentences) | |
| Format your response as a JSON array with this structure: | |
| [ | |
| { | |
| "title": "Discussion title", | |
| "classification": "BUG_REPORT", | |
| "confidence": 85, | |
| "reasoning": "The title mentions 'empty outputs' which suggests a functionality issue or bug." | |
| } | |
| ] | |
| Here are the discussions to analyze: | |
| EOF | |
| # Add discussions to prompt | |
| echo "$DISCUSSION_LIST" >> analysis_prompt.txt | |
| # Read the prompt | |
| PROMPT=$(cat analysis_prompt.txt) | |
| # Call GitHub Models API with GPT-4o | |
| echo "🤖 Analyzing discussions with AI..." | |
| # Check if we have a token (simplified) | |
| if [ -z "$GITHUB_TOKEN" ]; then | |
| echo "ERROR: GITHUB_TOKEN is not set" | |
| exit 1 | |
| fi | |
| # Use the correct API endpoint and model format | |
| AI_RESPONSE=$(curl -s -w "\nHTTP_STATUS:%{http_code}\n" "https://models.github.ai/inference/chat/completions" \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Content-Type: application/json" \ | |
| -H "Authorization: Bearer $GITHUB_TOKEN" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| -d "{ | |
| \"model\": \"openai/gpt-4o\", | |
| \"messages\": [ | |
| { | |
| \"role\": \"user\", | |
| \"content\": $(echo "$PROMPT" | jq -Rs .) | |
| } | |
| ] | |
| }") | |
| # Extract HTTP status and response | |
| HTTP_STATUS=$(echo "$AI_RESPONSE" | grep "HTTP_STATUS:" | cut -d: -f2) | |
| API_RESPONSE=$(echo "$AI_RESPONSE" | sed '/HTTP_STATUS:/d') | |
| # Check if the request was successful | |
| if [ "$HTTP_STATUS" != "200" ]; then | |
| echo "❌ AI API request failed. Using fallback analysis..." | |
| # Fallback to keyword-based analysis | |
| AI_ANALYSIS=$(echo "$DISCUSSIONS" | jq -r '[.[] | { | |
| "title": .title, | |
| "classification": (if (.title | ascii_downcase | test("bug|error|issue|problem|broken|not working|fail|crash|exception|wrong|incorrect")) then "BUG_REPORT" | |
| elif (.title | ascii_downcase | test("feature|request|enhancement|improvement|add|support")) then "FEATURE_REQUEST" | |
| elif (.title | ascii_downcase | test("how to|question|help|documentation|guide|usage")) then "QUESTION" | |
| else "DISCUSSION" end), | |
| "confidence": (if (.title | ascii_downcase | test("bug|error|crash")) then 80 | |
| elif (.title | ascii_downcase | test("issue|problem|broken")) then 70 | |
| elif (.title | ascii_downcase | test("feature|request")) then 75 | |
| elif (.title | ascii_downcase | test("how to|question")) then 65 | |
| else 50 end), | |
| "reasoning": "Classified using keyword analysis." | |
| }]') | |
| else | |
| # Extract AI response content and clean it | |
| AI_ANALYSIS=$(echo "$API_RESPONSE" | jq -r '.choices[0].message.content' | sed 's/```json//g' | sed 's/```//g' | sed '/^$/d') | |
| fi | |
| echo "" | |
| echo "========================================" | |
| echo "🤖 GITHUB DISCUSSIONS ANALYSIS REPORT" | |
| echo "========================================" | |
| echo "" | |
| # Parse AI analysis and create summary | |
| if echo "$AI_ANALYSIS" | jq -e . >/dev/null 2>&1; then | |
| # Get counts for each category | |
| BUG_COUNT=$(echo "$AI_ANALYSIS" | jq '[.[] | select(.classification == "BUG_REPORT")] | length') | |
| FEATURE_COUNT=$(echo "$AI_ANALYSIS" | jq '[.[] | select(.classification == "FEATURE_REQUEST")] | length') | |
| QUESTION_COUNT=$(echo "$AI_ANALYSIS" | jq '[.[] | select(.classification == "QUESTION")] | length') | |
| DISCUSSION_COUNT=$(echo "$AI_ANALYSIS" | jq '[.[] | select(.classification == "DISCUSSION")] | length') | |
| TOTAL_COUNT=$(echo "$AI_ANALYSIS" | jq 'length') | |
| echo "📊 SUMMARY (Last 24 Hours)" | |
| echo "─────────────────────────" | |
| echo "�🐛 Bug Reports: $BUG_COUNT" | |
| echo "✨ Feature Requests: $FEATURE_COUNT" | |
| echo "❓ Questions: $QUESTION_COUNT" | |
| echo "💬 General Discussions: $DISCUSSION_COUNT" | |
| echo "📝 Total Discussions: $TOTAL_COUNT" | |
| echo "" | |
| # Show bug reports section | |
| echo "🐛 BUG REPORTS DETECTED" | |
| echo "─────────────────────────" | |
| if [ "$BUG_COUNT" -gt 0 ]; then | |
| # Get bug report titles and add URLs from original discussions | |
| echo "$AI_ANALYSIS" | jq -r '.[] | select(.classification == "BUG_REPORT") | .title' | while read -r title; do | |
| confidence=$(echo "$AI_ANALYSIS" | jq -r --arg title "$title" '.[] | select(.title == $title) | .confidence') | |
| reasoning=$(echo "$AI_ANALYSIS" | jq -r --arg title "$title" '.[] | select(.title == $title) | .reasoning') | |
| url=$(echo "$DISCUSSIONS" | jq -r --arg title "$title" '.[] | select(.title == $title) | .url') | |
| echo "• $title ($confidence% confidence)" | |
| echo " 💭 $reasoning" | |
| echo " 🔗 $url" | |
| echo "" | |
| done | |
| else | |
| echo "✅ No bug reports identified in recent discussions." | |
| echo "" | |
| fi | |
| # Show all other discussions organized | |
| echo "📋 OTHER RECENT DISCUSSIONS" | |
| echo "─────────────────────────────" | |
| # Feature Requests | |
| if [ "$FEATURE_COUNT" -gt 0 ]; then | |
| echo "✨ Feature Requests:" | |
| echo "$AI_ANALYSIS" | jq -r '.[] | select(.classification == "FEATURE_REQUEST") | .title' | while read -r title; do | |
| confidence=$(echo "$AI_ANALYSIS" | jq -r --arg title "$title" '.[] | select(.title == $title) | .confidence') | |
| url=$(echo "$DISCUSSIONS" | jq -r --arg title "$title" '.[] | select(.title == $title) | .url') | |
| echo " • $title ($confidence% confidence)" | |
| echo " 🔗 $url" | |
| done | |
| echo "" | |
| fi | |
| # Questions | |
| if [ "$QUESTION_COUNT" -gt 0 ]; then | |
| echo "❓ Questions:" | |
| echo "$AI_ANALYSIS" | jq -r '.[] | select(.classification == "QUESTION") | .title' | while read -r title; do | |
| confidence=$(echo "$AI_ANALYSIS" | jq -r --arg title "$title" '.[] | select(.title == $title) | .confidence') | |
| url=$(echo "$DISCUSSIONS" | jq -r --arg title "$title" '.[] | select(.title == $title) | .url') | |
| echo " • $title ($confidence% confidence)" | |
| echo " 🔗 $url" | |
| done | |
| echo "" | |
| fi | |
| # General Discussions | |
| if [ "$DISCUSSION_COUNT" -gt 0 ]; then | |
| echo "💬 General Discussions:" | |
| echo "$AI_ANALYSIS" | jq -r '.[] | select(.classification == "DISCUSSION") | .title' | while read -r title; do | |
| confidence=$(echo "$AI_ANALYSIS" | jq -r --arg title "$title" '.[] | select(.title == $title) | .confidence') | |
| url=$(echo "$DISCUSSIONS" | jq -r --arg title "$title" '.[] | select(.title == $title) | .url') | |
| echo " • $title ($confidence% confidence)" | |
| echo " 🔗 $url" | |
| done | |
| echo "" | |
| fi | |
| else | |
| echo "❌ Could not parse AI analysis. Raw response:" | |
| echo "$AI_ANALYSIS" | |
| fi | |
| echo "" | |
| echo "========================================" | |
| # Save AI analysis for next step | |
| echo "$AI_ANALYSIS" > ai_analysis.json | |
| # Clean up | |
| rm -f analysis_prompt.txt | |
| else | |
| echo "No discussions.json file found. Skipping analysis." | |
| fi | |
| - name: Create GitHub issues for bug reports | |
| if: always() | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| // Check if analysis file exists | |
| if (!fs.existsSync('ai_analysis.json')) { | |
| console.log('No AI analysis file found. Skipping issue creation.'); | |
| return; | |
| } | |
| // Read the AI analysis results | |
| let aiAnalysis; | |
| try { | |
| const aiData = fs.readFileSync('ai_analysis.json', 'utf8'); | |
| aiAnalysis = JSON.parse(aiData); | |
| } catch (error) { | |
| console.log('Could not parse AI analysis file. Skipping issue creation.'); | |
| console.log('Error:', error.message); | |
| return; | |
| } | |
| // Read original discussions data | |
| let discussions = []; | |
| if (fs.existsSync('discussions.json')) { | |
| try { | |
| const discussionData = fs.readFileSync('discussions.json', 'utf8'); | |
| discussions = JSON.parse(discussionData); | |
| } catch (error) { | |
| console.log('Could not parse discussions file:', error.message); | |
| } | |
| } | |
| // Filter for bug reports | |
| const bugReports = aiAnalysis.filter(item => item.classification === 'BUG_REPORT'); | |
| if (bugReports.length === 0) { | |
| console.log('✅ No bug reports detected. No issues to create.'); | |
| return; | |
| } | |
| console.log(`🐛 Creating ${bugReports.length} GitHub issues for detected bug reports...`); | |
| // Create issues for each bug report | |
| for (const bugReport of bugReports) { | |
| try { | |
| // Find the corresponding discussion for additional details | |
| const originalDiscussion = discussions.find(d => d.title === bugReport.title); | |
| // Create issue title | |
| const issueTitle = `[Auto-detected Bug] ${bugReport.title}`; | |
| // Create issue body | |
| const issueBody = `## Auto-detected Bug Report | |
| This issue was automatically created based on AI analysis of a GitHub discussion in the Models category. | |
| **Original Discussion:** ${originalDiscussion ? originalDiscussion.url : 'URL not found'} | |
| **AI Confidence:** ${bugReport.confidence}% | |
| **Analysis:** ${bugReport.reasoning} | |
| ### Discussion Details | |
| ${originalDiscussion ? ` | |
| - **Author:** ${originalDiscussion.author} | |
| - **Date:** ${originalDiscussion.timeText} | |
| - **Comments:** ${originalDiscussion.commentCount} | |
| ` : 'Details not available'} | |
| ### Next Steps | |
| - [ ] Review the original discussion for more context | |
| - [ ] Verify if this is actually a bug | |
| - [ ] Determine if this is a duplicate of an existing issue | |
| - [ ] Take appropriate action (fix, document, close, etc.) | |
| --- | |
| *This issue was automatically created by the GitHub Models Discussion Watcher workflow.*`; | |
| // Create the issue | |
| const issue = await github.rest.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title: issueTitle, | |
| body: issueBody, | |
| labels: ['auto-detected', 'bug-report', 'needs-triage'] | |
| }); | |
| console.log(`✅ Created issue #${issue.data.number}: ${issueTitle}`); | |
| } catch (error) { | |
| console.log(`❌ Failed to create issue for "${bugReport.title}":`, error.message); | |
| } | |
| } | |
| console.log('🎉 Issue creation process completed!'); |