diff --git a/.github/workflows/review.yml b/.github/workflows/review.yml index 93b01bafa2b..6051103f0ef 100644 --- a/.github/workflows/review.yml +++ b/.github/workflows/review.yml @@ -49,12 +49,15 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} OPENCODE_PERMISSION: '{ "bash": { "*": "deny", "gh*": "allow", "gh pr review*": "deny" } }' PR_TITLE: ${{ steps.pr-details.outputs.title }} + REVIEW_PROVIDERS: "opencode/big-pickle,opencode/grok-code,opencode/minimax-m2.1-free,opencode/glm-4.7-free" run: | PR_BODY=$(jq -r .body pr_data.json) - opencode run -m anthropic/claude-opus-4-5 "A new pull request has been created: '${PR_TITLE}' + PR_NUMBER="${{ steps.pr-number.outputs.number }}" + + export REVIEW_PROMPT="A new pull request has been created: '${PR_TITLE}' - ${{ steps.pr-number.outputs.number }} + $PR_NUMBER @@ -64,20 +67,22 @@ jobs: Please check all the code changes in this pull request against the style guide, also look for any bugs if they exist. Diffs are important but make sure you read the entire file to get proper context. Make it clear the suggestions are merely suggestions and the human can decide what to do When critiquing code against the style guide, be sure that the code is ACTUALLY in violation, don't complain about else statements if they already use early returns there. You may complain about excessive nesting though, regardless of else statement usage. - When critiquing code style don't be a zealot, we don't like "let" statements but sometimes they are the simplest option, if someone does a bunch of nesting with let, they should consider using iife (see packages/opencode/src/util.iife.ts) + When critiquing code style don't be a zealot, we don't like \"let\" statements but sometimes they are the simplest option, if someone does a bunch of nesting with let, they should consider using iife (see packages/opencode/src/util.iife.ts) Use the gh cli to create comments on the files for the violations. Try to leave the comment on the exact line number. If you have a suggested fix include it in a suggestion code block. - If you are writing suggested fixes, BE SURE THAT the change you are recommending is actually valid typescript, often I have seen missing closing "}" or other syntax errors. + If you are writing suggested fixes, BE SURE THAT the change you are recommending is actually valid typescript, often I have seen missing closing \"}\" or other syntax errors. Generally, write a comment instead of writing suggested change if you can help it. Command MUST be like this. \`\`\` - gh api \ - --method POST \ - -H \"Accept: application/vnd.github+json\" \ - -H \"X-GitHub-Api-Version: 2022-11-28\" \ - /repos/${{ github.repository }}/pulls/${{ steps.pr-number.outputs.number }}/comments \ + gh api \\ + --method POST \\ + -H \"Accept: application/vnd.github+json\" \\ + -H \"X-GitHub-Api-Version: 2022-11-28\" \\ + /repos/${{ github.repository }}/pulls/$PR_NUMBER/comments \\ -f 'body=[summary of issue]' -f 'commit_id=${{ steps.pr-details.outputs.sha }}' -f 'path=[path-to-file]' -F \"line=[line]\" -f 'side=RIGHT' \`\`\` Only create comments for actual violations. If the code follows all guidelines, comment on the issue using gh cli: 'lgtm' AND NOTHING ELSE!!!!." + + bun run github/multi-review.ts diff --git a/github/MULTI_REVIEW.md b/github/MULTI_REVIEW.md new file mode 100644 index 00000000000..62437023485 --- /dev/null +++ b/github/MULTI_REVIEW.md @@ -0,0 +1,49 @@ +# Multi-Provider Code Review + +This feature allows the GitHub Action to run code reviews simultaneously with multiple AI providers and then synthesize their results into a comprehensive review. + +## How It Works + +1. **Parallel Reviews**: The workflow runs reviews with multiple providers simultaneously using `Promise.all` +2. **Aggregation**: All individual reviews are collected and formatted +3. **Synthesis**: A designated synthesis model combines overlapping feedback, highlights unique insights, and removes duplicates + +## Configuration + +The providers are configured via the `REVIEW_PROVIDERS` environment variable in `.github/workflows/review.yml`: + +```yaml +REVIEW_PROVIDERS: "opencode/big-pickle,opencode/grok-code,opencode/minimax-m2.1-free,opencode/glm-4.7-free" +``` + +### Current Free Providers + +- `opencode/big-pickle` - Large reasoning model +- `opencode/grok-code` - Code-specialized model +- `opencode/minimax-m2.1-free` - Free tier model +- `opencode/glm-4.7-free` - Free GLM model + +## Adding New Providers + +To add or change providers: + +1. Edit the `REVIEW_PROVIDERS` variable in `.github/workflows/review.yml` +2. Use the format `provider/model` (e.g., `opencode/big-pickle`) +3. Separate multiple providers with commas + +## Script Details + +The `github/multi-review.ts` script handles: + +- Running parallel reviews with error handling +- 5-minute timeout per provider +- Graceful fallback if a provider fails +- Synthesis using the first provider in the list +- Proper logging and status reporting + +## Benefits + +- **Comprehensive Reviews**: Multiple perspectives catch different issues +- **Redundancy**: If one provider fails, others continue +- **Cost Efficiency**: Uses free providers +- **Quality Synthesis**: Combines the best insights from all reviews diff --git a/github/multi-provider-review-template.yml b/github/multi-provider-review-template.yml new file mode 100644 index 00000000000..81e7be3bdd7 --- /dev/null +++ b/github/multi-provider-review-template.yml @@ -0,0 +1,87 @@ +name: ๐Ÿค– Multi-Provider Code Review + +on: + pull_request: + types: [opened, synchronize, reopened] + issue_comment: + types: [created] + +jobs: + check-guidelines: + if: | + (github.event_name == 'pull_request') || + (github.event.issue.pull_request && + (startsWith(github.event.comment.body, '/review') || + contains(github.event.comment.body, '@opencode') || + contains(github.event.comment.body, '@claude'))) + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - name: Get PR number + id: pr-number + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "number=${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT + else + echo "number=${{ github.event.issue.number }}" >> $GITHUB_OUTPUT + fi + + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Setup Bun + uses: oven-sh/setup-bun@v1 + with: + bun-version: latest + + - name: Install opencode + run: curl -fsSL https://opencode.ai/install | bash + + - name: Get PR details + id: pr-details + run: | + gh api /repos/${{ github.repository }}/pulls/${{ steps.pr-number.outputs.number }} > pr_data.json + echo "title=$(jq -r .title pr_data.json)" >> $GITHUB_OUTPUT + echo "sha=$(jq -r .head.sha pr_data.json)" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Check for AGENTS.md + id: agents-md + run: | + if [ -f AGENTS.md ]; then + echo "exists=true" >> $GITHUB_OUTPUT + else + echo "exists=false" >> $GITHUB_OUTPUT + fi + + - name: Get PR body + id: pr-body + run: | + PR_BODY=$(gh api /repos/${{ github.repository }}/pulls/${{ steps.pr-number.outputs.number }} --jq .body | tr '\n' ' ' | sed 's/"/\\"/g') + echo "body<> $GITHUB_OUTPUT + echo "$PR_BODY" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Run Multi-Provider Review + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REVIEW_PROVIDERS: ${{ vars.REVIEW_PROVIDERS || 'opencode/big-pickle,opencode/grok-code,opencode/minimax-m2.1-free,opencode/glm-4.7-free' }} + PR_TITLE: ${{ steps.pr-details.outputs.title }} + PR_NUMBER: ${{ steps.pr-number.outputs.number }} + PR_BODY: ${{ steps.pr-body.outputs.body }} + HAS_AGENTS: ${{ steps.agents-md.outputs.exists || 'false' }} + run: | + cp github/multi-review-script.ts multi-review.ts + + if [ "$HAS_AGENTS" = "true" ]; then + export INCLUDE_AGENTS=true + fi + + bun run multi-review.ts diff --git a/github/multi-provider-review.yml b/github/multi-provider-review.yml new file mode 100644 index 00000000000..a01f5b0f7f8 --- /dev/null +++ b/github/multi-provider-review.yml @@ -0,0 +1,91 @@ +name: Multi-Provider Code Review + +on: + pull_request: + types: [opened, synchronize, reopened] + issue_comment: + types: [created] + +jobs: + review: + if: | + (github.event_name == 'pull_request') || + (github.event.issue.pull_request && + (startsWith(github.event.comment.body, '/review') || + contains(github.event.comment.body, '@opencode') || + contains(github.event.comment.body, '@claude'))) + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - name: Get PR number + id: pr-number + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "number=${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT + else + echo "number=${{ github.event.issue.number }}" >> $GITHUB_OUTPUT + fi + + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Setup Bun + uses: oven-sh/setup-bun@v1 + with: + bun-version: latest + + - name: Install opencode + run: | + for i in {1..3}; do + echo "Attempt $i of 3 to install opencode..." + if curl -fsSL https://opencode.ai/install | bash; then + echo "opencode installed successfully" + exit 0 + fi + echo "Attempt $i failed, waiting 5 seconds before retry..." + sleep 5 + done + echo "Failed to install opencode after 3 attempts" + exit 1 + + - name: Get PR details + id: pr-details + env: + GH_TOKEN: ${{ github.token }} + run: | + gh api /repos/${{ github.repository }}/pulls/${{ steps.pr-number.outputs.number }} > pr_data.json + echo "title=$(jq -r .title pr_data.json)" >> $GITHUB_OUTPUT + echo "sha=$(jq -r .head.sha pr_data.json)" >> $GITHUB_OUTPUT + + - name: Get PR body + id: pr-body + env: + GH_TOKEN: ${{ github.token }} + run: | + PR_BODY=$(gh api /repos/${{ github.repository }}/pulls/${{ steps.pr-number.outputs.number }} --jq .body | tr '\n' ' ' | sed 's/"/\\"/g') + echo "body<> $GITHUB_OUTPUT + echo "$PR_BODY" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - name: Check for AGENTS.md + id: agents-md + run: | + if [ -f AGENTS.md ]; then + echo "exists=true" >> $GITHUB_OUTPUT + else + echo "exists=false" >> $GITHUB_OUTPUT + fi + + - name: Run Multi-Provider Review + uses: keithah/multi-provider-code-review@main + with: + GITHUB_TOKEN: ${{ github.token }} + REVIEW_PROVIDERS: ${{ vars.REVIEW_PROVIDERS || 'opencode/big-pickle,opencode/grok-code,opencode/minimax-m2.1-free,opencode/glm-4.7-free' }} + PR_TITLE: ${{ steps.pr-details.outputs.title }} + PR_NUMBER: ${{ steps.pr-number.outputs.number }} + PR_BODY: ${{ steps.pr-body.outputs.body }} + HAS_AGENTS: ${{ steps.agents-md.outputs.exists || 'false' }} diff --git a/github/multi-review-script.ts b/github/multi-review-script.ts new file mode 100644 index 00000000000..44738848b18 --- /dev/null +++ b/github/multi-review-script.ts @@ -0,0 +1,308 @@ +import { execSync } from "child_process" +import { existsSync, readFileSync } from "fs" + +type ReviewResult = { + provider: string + output: string +} + +const REVIEW_PROVIDERS = (process.env.REVIEW_PROVIDERS || "") + .split(",") + .map((p) => p.trim()) + .filter(Boolean) +const PR_NUMBER = process.env.PR_NUMBER || "" +const PR_TITLE = process.env.PR_TITLE || "" +const PR_BODY = process.env.PR_BODY || "" +const REPO = process.env.GITHUB_REPOSITORY || "" +const INCLUDE_AGENTS = process.env.INCLUDE_AGENTS === "true" + +async function buildPrompt(): Promise { + let prBody = PR_BODY || "" + console.log(`Using PR body: ${prBody.substring(0, 100)}...`) + + let agentsSection = "" + try { + if (INCLUDE_AGENTS && existsSync("AGENTS.md")) { + const agentsContent = readFileSync("AGENTS.md", "utf8").substring(0, 2000) + agentsSection = `\n\n## Project Guidelines (from AGENTS.md)\n${agentsContent}` + } + } catch (e) { + console.log("Warning: Could not read AGENTS.md") + } + + return `REPO: ${REPO} +PR NUMBER: ${PR_NUMBER} +PR TITLE: ${PR_TITLE} +PR DESCRIPTION: +${prBody} + +Please review this pull request and provide comprehensive code review focusing on: + +## Code Quality & Best Practices +- Clean code principles and readability +- Proper error handling and edge cases +- TypeScript/JavaScript best practices +- Consistent naming conventions + +## Bug Detection +- Logic errors and edge cases +- Unhandled error scenarios +- Race conditions and concurrency issues +- Input validation and sanitization + +## Performance +- Inefficient algorithms or operations +- Memory leaks and unnecessary allocations +- Large file handling + +## Security +- SQL injection, XSS, CSRF vulnerabilities +- Authentication/authorization issues +- Sensitive data exposure + +## Testing +- Test coverage gaps +- Missing edge case handling${agentsSection} + +## Output Format +- Use \`gh pr comment\` to leave review comments on specific files +- Include specific line numbers and code suggestions +- Provide actionable recommendations +- Summarize key findings at the end + +IMPORTANT: Only create comments for actual issues. If the code follows all guidelines, respond with 'lgtm' only.` +} + +async function runReview(provider: string, prompt: string): Promise { + console.log(`Starting review with provider: ${provider}`) + + try { + const result = execSync(`opencode run -m ${provider} -- "${prompt.replace(/"/g, '\\"')}"`, { + encoding: "utf8", + timeout: 180000, + }) + return { provider, output: result } + } catch (error) { + console.error(`Error with ${provider}:`, error) + return { provider, output: `Error: Review failed for ${provider}` } + } +} + +async function synthesize(reviews: ReviewResult[]): Promise { + const combined = reviews.map((r) => `## Review from ${r.provider}\n\n${r.output}`).join("\n\n---\n\n") + const providerList = reviews.map((r) => r.provider).join(", ") + const synthesisPrompt = `You are an expert code reviewer. Synthesize these reviews into one comprehensive review following Claude Code's professional format. + +Rules: +- Combine overlapping feedback and remove duplicates +- Highlight unique insights from each review +- Present a clear, actionable review with professional formatting +- Include effort estimation (1-5 scale) and size labels +- Add checklists for different concern categories +- Provide specific code suggestions with line numbers +- Include security, performance, and testing recommendations +- Structure like Claude Code: summary table, detailed analysis, code suggestions + +Reviews from providers: ${providerList} + +Reviews to synthesize: +${combined} + +Output Format (match Claude Code exact style): + +**Summary** +[Brief summary of PR and overall findings] + +**Critical Issues** โš ๏ธ +1. โš ๏ธ **[Issue Title] ([Severity] - [Category])** +Location: [file]:[line-range] + +[Detailed description of critical issue] + +[Code example if relevant] + +Recommendation: [Specific fix suggestion] + +\`\`\`typescript +// Suggested fix +[fixed code] +\`\`\` + +**Code Quality Issues** โœ… +[Number]. โœ… **Positive: [Title]** - [Description] + +[Number]. โš ๏ธ **[Issue Title]** - [Description] +Location: [file]:[line-range] + +[Code example] + +Recommendation: [Fix suggestion] + +**Testing Recommendations** ๐Ÿงช +Before merging, please test with: + +โœ… [Test case 1] +โš ๏ธ [Test case 2 - requires attention] +โœ… [Test case 3] + +**Verdict** +Status: โš ๏ธ [Approve with Recommendations / Changes Requested / Approved] + +[Overall assessment and recommendations] + +**Positive Aspects** โœจ +โœ… [Positive aspect 1] +โœ… [Positive aspect 2] +โœ… [Positive aspect 3]` + + try { + return execSync(`opencode run -m opencode/big-pickle -- "${synthesisPrompt.replace(/"/g, '\\"')}"`, { + encoding: "utf8", + timeout: 180000, + }) + } catch (error) { + console.error("Synthesis error:", error) + return combined + } +} + +let checklistCommentId: string | null = null + +async function postOrUpdateChecklist(status: string, completedTasks: string[] = []) { + const tasks = [ + "Read repository conventions", + "Read modified files", + "Analyze security implications", + "Review code quality and conventions", + "Provide comprehensive feedback", + ] + + const checklist = tasks + .map((task) => { + const isCompleted = completedTasks.includes(task) + return `${isCompleted ? "โœ…" : "โณ"} ${task}` + }) + .join("\n") + + const body = `๐Ÿค– **Multi-Provider Code Review** ${status}\n\n**Tasks:**\n${checklist}` + + const escaped = body.replace(/"/g, '\\"').replace(/\n/g, "\\n") + + if (checklistCommentId) { + execSync( + `gh api --method PATCH -H "Accept: application/vnd.github+json" /repos/${REPO}/issues/comments/${checklistCommentId} -f "body=${escaped}"`, + { + encoding: "utf8", + }, + ) + } else { + const result = execSync( + `gh api --method POST -H "Accept: application/vnd.github+json" /repos/${REPO}/issues/${PR_NUMBER}/comments -f "body=${escaped}"`, + { + encoding: "utf8", + }, + ) + const comment = JSON.parse(result.toString()) + checklistCommentId = comment.id + } +} + +async function postFinalReview(synthesis: string, providerList: string, confidenceString: string) { + const finalBody = `๐Ÿค– **Code Review Complete** + +**Tasks:** +โœ… Read repository conventions +โœ… Read modified files +โœ… Analyze security implications +โœ… Review code quality and conventions +โœ… Provide comprehensive feedback + +${synthesis} + +*Review generated by: ${providerList}* +*Provider confidence scores: ${confidenceString}*` + + const escaped = finalBody.replace(/"/g, '\\"').replace(/\n/g, "\\n") + execSync( + `gh api --method POST -H "Accept: application/vnd.github+json" /repos/${REPO}/issues/${PR_NUMBER}/comments -f "body=${escaped}"`, + { + encoding: "utf8", + }, + ) +} + +async function calculateConfidenceScores(reviews: ReviewResult[]): Promise> { + const scores: Record = {} + + for (const review of reviews) { + let score = 0.5 + + if (review.output.length > 2000) score += 0.2 + else if (review.output.length > 1000) score += 0.1 + + if (review.output.includes("security") || review.output.includes("performance")) score += 0.1 + if (review.output.includes("suggestion") || review.output.includes("recommend")) score += 0.1 + if (review.output.includes("line") || review.output.includes("file")) score += 0.1 + + if (review.provider.includes("big-pickle")) score += 0.1 + if (review.provider.includes("grok-code")) score += 0.1 + if (review.provider.includes("minimax")) score += 0.05 + if (review.provider.includes("glm-4.7")) score += 0.05 + + scores[review.provider] = Math.min(Math.max(score, 0.1), 1.0) + } + + return scores +} + +async function main() { + console.log(`Running reviews with ${REVIEW_PROVIDERS.length} providers`) + + const prompt = await buildPrompt() + console.log(`Prompt built (${prompt.length} chars), includes AGENTS.md: ${INCLUDE_AGENTS}`) + + await postOrUpdateChecklist("๐Ÿ” Reading modified files...", ["Read repository conventions"]) + + await postOrUpdateChecklist("๐Ÿ” Analyzing security implications...", [ + "Read repository conventions", + "Read modified files", + "Analyze security implications", + ]) + + const results = await Promise.all(REVIEW_PROVIDERS.map((provider) => runReview(provider, prompt))) + + await postOrUpdateChecklist("๐Ÿ” Reviewing code quality and conventions...", [ + "Read repository conventions", + "Read modified files", + "Analyze security implications", + "Review code quality and conventions", + ]) + + console.log("\nAll reviews completed. Synthesizing...") + + await postOrUpdateChecklist("๐Ÿค” Providing comprehensive feedback...", [ + "Read repository conventions", + "Read modified files", + "Analyze security implications", + "Review code quality and conventions", + "Provide comprehensive feedback", + ]) + + const synthesis = await synthesize(results) + const providerList = results.map((r) => r.provider).join(", ") + const confidenceScores = await calculateConfidenceScores(results) + const confidenceString = Object.entries(confidenceScores) + .map(([provider, score]) => `${provider}: ${(score * 100).toFixed(0)}%`) + .join(", ") + + console.log("\n=== SYNTHESIS COMPLETE ===\n") + console.log(synthesis) + + await postFinalReview(synthesis, providerList, confidenceString) + console.log("\nโœ… Final review posted to PR!") +} + +main().catch((error) => { + console.error("Error:", error) + process.exit(1) +}) diff --git a/github/multi-review.ts b/github/multi-review.ts new file mode 100644 index 00000000000..e1c1605d239 --- /dev/null +++ b/github/multi-review.ts @@ -0,0 +1,95 @@ +import { execSync } from "child_process" + +type ReviewResult = { + provider: string + output: string +} + +const REVIEW_PROVIDERS = process.env.REVIEW_PROVIDERS?.split(",").map((p) => p.trim()) || [] +const REVIEW_PROMPT = process.env.REVIEW_PROMPT || "" +// ANTHROPIC_API_KEY is optional for testing + +if (REVIEW_PROVIDERS.length === 0) { + console.error("REVIEW_PROVIDERS not set") + process.exit(1) +} + +if (!REVIEW_PROMPT) { + console.error("REVIEW_PROMPT not set") + process.exit(1) +} + +async function runReview(provider: string): Promise { + console.log(`Starting review with provider: ${provider}`) + + try { + const result = execSync(`opencode run -m ${provider} -- "${REVIEW_PROMPT}"`, { + encoding: "utf8", + timeout: 300000, // 5 minutes timeout + }) + return { + provider, + output: result, + } + } catch (error) { + console.error(`Error running review with ${provider}:`, error) + return { + provider, + output: `Error: Review failed for ${provider}`, + } + } +} + +async function aggregateReviews(results: ReviewResult[]): Promise { + const aggregated = results.map((r) => `## Review from ${r.provider}\n\n${r.output}`).join("\n\n---\n\n") + + const synthesisPrompt = `You are an expert code reviewer. Please synthesize the following reviews from multiple AI models into a single, comprehensive code review. + +Rules: +- Combine overlapping feedback +- Highlight unique insights from each review +- Remove duplicates +- Present a clear, actionable review +- Maintain the tone and suggestions from the reviews +- If all reviews agree on something, state that clearly +- Keep the synthesized review concise and actionable +- Include all the specific code violations found by any reviewer + +Here are the reviews to synthesize: + +${aggregated} + +Please provide the synthesized review:` + + try { + const synthesisResult = execSync(`opencode run -m opencode/big-pickle -- "${synthesisPrompt}"`, { + encoding: "utf8", + timeout: 300000, // 5 minutes timeout + }) + return synthesisResult + } catch (error) { + console.error("Error synthesizing reviews:", error) + return aggregated + } +} + +async function main() { + console.log(`Running reviews with ${REVIEW_PROVIDERS.length} providers: ${REVIEW_PROVIDERS.join(", ")}`) + console.log(`Prompt length: ${REVIEW_PROMPT.length} characters`) + + const results = await Promise.all(REVIEW_PROVIDERS.map((provider) => runReview(provider))) + + console.log("\nAll reviews completed. Aggregating...") + + const synthesis = await aggregateReviews(results) + + console.log("\n=== SYNTHESIZED REVIEW ===\n") + console.log(synthesis) + + process.stdout.write(synthesis) +} + +main().catch((error) => { + console.error("Error:", error) + process.exit(1) +})