This repository was archived by the owner on Aug 10, 2026. It is now read-only.
Production Health Check #88
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright © 2024–2026 Jasper Ford | |
| # SPDX-License-Identifier: AGPL-3.0-or-later | |
| name: Production Health Check | |
| on: | |
| schedule: | |
| - cron: '*/30 * * * *' | |
| workflow_dispatch: | |
| jobs: | |
| health: | |
| name: Check Production Health | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check API health | |
| id: api_health | |
| run: | | |
| HTTP_STATUS=$(curl -s -o /tmp/health_response.txt -w "%{http_code}" \ | |
| --max-time 15 \ | |
| https://api.project-nexus.net/health || echo "000") | |
| BODY=$(cat /tmp/health_response.txt 2>/dev/null || echo "no response") | |
| echo "status=$HTTP_STATUS" >> $GITHUB_OUTPUT | |
| echo "body=$BODY" >> $GITHUB_OUTPUT | |
| if [ "$HTTP_STATUS" != "200" ]; then | |
| echo "::error::API health check failed with HTTP $HTTP_STATUS: $BODY" | |
| fi | |
| - name: Check UK frontend | |
| id: uk_health | |
| run: | | |
| HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ | |
| --max-time 15 \ | |
| https://uk.project-nexus.net || echo "000") | |
| echo "status=$HTTP_STATUS" >> $GITHUB_OUTPUT | |
| if [ "$HTTP_STATUS" != "200" ]; then | |
| echo "::warning::UK frontend returned HTTP $HTTP_STATUS" | |
| fi | |
| - name: Check Modern frontend | |
| id: app_health | |
| run: | | |
| HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ | |
| --max-time 15 \ | |
| https://app.project-nexus.net || echo "000") | |
| echo "status=$HTTP_STATUS" >> $GITHUB_OUTPUT | |
| if [ "$HTTP_STATUS" != "200" ]; then | |
| echo "::warning::Modern frontend returned HTTP $HTTP_STATUS" | |
| fi | |
| - name: Create issue on failure | |
| if: steps.api_health.outputs.status != '200' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const title = `Production health check failed - API returned HTTP ${process.env.API_STATUS}`; | |
| const body = [ | |
| '## Production Health Alert', | |
| '', | |
| `**Time**: ${new Date().toISOString()}`, | |
| `**API Status**: HTTP ${process.env.API_STATUS}`, | |
| `**UK Frontend**: HTTP ${process.env.UK_STATUS}`, | |
| `**Modern Frontend**: HTTP ${process.env.APP_STATUS}`, | |
| '', | |
| '### Recovery Steps', | |
| '1. SSH to production: `ssh -i key.pem azureuser@<server-ip>`', | |
| '2. Check containers: `cd /opt/nexus-backend && sudo docker compose ps`', | |
| '3. Check API logs: `sudo docker compose logs --tail 50 api`', | |
| '4. Restart if needed: `sudo docker compose restart api`', | |
| '5. Consult RECOVERY_GUIDE.md for detailed steps', | |
| '', | |
| '_This issue was auto-generated by the health check workflow._' | |
| ].join('\n'); | |
| const { data: issues } = await github.rest.issues.listForRepo({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| state: 'open', | |
| labels: 'health-alert' | |
| }); | |
| if (issues.length === 0) { | |
| await github.rest.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title: title, | |
| body: body, | |
| labels: ['health-alert', 'urgent'] | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: issues[0].number, | |
| body: `Health check still failing at ${new Date().toISOString()} — API: HTTP ${process.env.API_STATUS}` | |
| }); | |
| } | |
| env: | |
| API_STATUS: ${{ steps.api_health.outputs.status }} | |
| UK_STATUS: ${{ steps.uk_health.outputs.status }} | |
| APP_STATUS: ${{ steps.app_health.outputs.status }} |