[Submission] Sharonai #11
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Evaluate Submission | |
| on: | |
| issues: | |
| types: [opened] | |
| jobs: | |
| evaluate: | |
| # Only run on issues with 'submission' label | |
| if: contains(github.event.issue.labels.*.name, 'submission') | |
| runs-on: ubuntu-latest | |
| permissions: | |
| issues: write | |
| contents: write | |
| pull-requests: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: | | |
| pip install requests beautifulsoup4 anthropic | |
| - name: Extract URL and evaluate | |
| id: evaluate | |
| env: | |
| ISSUE_BODY: ${{ github.event.issue.body }} | |
| ISSUE_NUMBER: ${{ github.event.issue.number }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| run: | | |
| python scripts/evaluate_submission.py | |
| - name: Read evaluation outputs | |
| id: read_outputs | |
| run: | | |
| echo "score=$(cat evaluation_score.txt)" >> $GITHUB_OUTPUT | |
| if [ -f submission_data.json ]; then | |
| echo "has_data=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "has_data=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Post evaluation results | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const results = fs.readFileSync('evaluation_results.md', 'utf8'); | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: results | |
| }); | |
| - name: Add result label | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const score = parseInt(fs.readFileSync('evaluation_score.txt', 'utf8').trim()); | |
| let label = 'needs-review'; | |
| if (score === 3) { | |
| label = 'auto-approved'; | |
| } else if (score === 2) { | |
| label = 'needs-review'; | |
| } else { | |
| label = 'needs-review'; // Even rejected ones go to needs-review for admin decision | |
| } | |
| await github.rest.issues.addLabels({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| labels: [label] | |
| }); | |
| // Add info comment for low scores | |
| if (score < 2) { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: `⚠️ **Admin Review Required**\n\nThis submission scored ${score}/3 and needs manual verification.\n\n**To approve:** Comment \`/approve\` or \`/approve 3\` (to override score)\n**To reject:** Add the \`rejected\` label and close the issue` | |
| }); | |
| } | |
| - name: Upload submission data as artifact | |
| if: steps.read_outputs.outputs.has_data == 'true' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: submission-data-${{ github.event.issue.number }} | |
| path: submission_data.json | |
| retention-days: 90 | |
| - name: Create PR if score >= 2 | |
| if: steps.read_outputs.outputs.score >= 2 && steps.read_outputs.outputs.has_data == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| ISSUE_NUMBER: ${{ github.event.issue.number }} | |
| run: | | |
| python scripts/create_submission_pr.py |