Index Repositories #276
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Index Repositories | |
| permissions: | |
| contents: read | |
| issues: write | |
| on: | |
| # Manual trigger | |
| workflow_dispatch: | |
| inputs: | |
| force_full_reindex: | |
| description: "Force full reindex (ignore cache)" | |
| required: false | |
| default: "false" | |
| type: boolean | |
| # Webhook trigger (for release notifications) | |
| repository_dispatch: | |
| types: [midnight-release, reindex] | |
| # Daily schedule (every day at 6am UTC) | |
| schedule: | |
| - cron: "0 6 * * *" | |
| # Prevent multiple indexing runs at the same time | |
| concurrency: | |
| group: midnight-indexing | |
| cancel-in-progress: false # Let running jobs finish, skip new triggers | |
| jobs: | |
| index: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 # Fast mode: tarball + batch + incremental | |
| environment: CLOUDFLARE # Use the CLOUDFLARE environment for secrets | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| cache: "npm" | |
| cache-dependency-path: api/package-lock.json | |
| - name: Install dependencies | |
| working-directory: api | |
| run: npm ci | |
| - name: Run indexing | |
| id: indexing | |
| working-directory: api | |
| env: | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| GITHUB_TOKEN: ${{ secrets.MIDNIGHT_GITHUB_TOKEN }} | |
| FORCE_REINDEX: ${{ inputs.force_full_reindex || 'false' }} | |
| run: npm run index | |
| - name: Create job summary | |
| if: always() | |
| run: | | |
| echo "## 📊 Indexing Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ "${{ steps.indexing.outcome }}" == "success" ]; then | |
| echo "✅ **Status:** Success" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "❌ **Status:** Failed" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Triggered by:** ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY | |
| echo "**Run at:** $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> $GITHUB_STEP_SUMMARY | |
| # Notification job - runs after indexing completes | |
| notify: | |
| needs: index | |
| runs-on: ubuntu-latest | |
| if: failure() | |
| steps: | |
| - name: Create failure issue | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const title = `⚠️ Weekly Indexing Failed - ${new Date().toISOString().split('T')[0]}`; | |
| const body = `## Indexing Workflow Failed | |
| The weekly repository indexing job failed. | |
| **Workflow Run:** [View Details](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) | |
| **Triggered:** ${{ github.event_name }} | |
| **Time:** ${new Date().toISOString()} | |
| ### Next Steps | |
| 1. Check the workflow logs for errors | |
| 2. Verify API keys are still valid (OpenAI, Cloudflare) | |
| 3. Check if any repos have changed structure | |
| 4. Re-run manually after fixing | |
| /cc @${{ github.repository_owner }}`; | |
| // Check for existing open issue | |
| const issues = await github.rest.issues.listForRepo({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| state: 'open', | |
| labels: 'indexing-failure' | |
| }); | |
| if (issues.data.length === 0) { | |
| await github.rest.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title: title, | |
| body: body, | |
| labels: ['indexing-failure', 'automated'] | |
| }); | |
| } |