Scrape Prices and Deploy #66
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scrape Prices and Deploy | |
| on: | |
| # Run daily at midnight UTC | |
| schedule: | |
| - cron: '0 0 * * *' | |
| # Allow manual trigger for testing | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| pages: write | |
| id-token: write | |
| jobs: | |
| scrape-and-build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| cache: 'npm' | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Install Chromium for Puppeteer | |
| run: | | |
| npx puppeteer browsers install chrome | |
| - name: Run all scrapers | |
| run: | | |
| node scrapers/run-all.js | |
| continue-on-error: true | |
| env: | |
| # Run in CI mode (headless, no sandbox) | |
| CI: true | |
| - name: Check scraper results | |
| id: scraper-status | |
| run: | | |
| if [ -d "data" ] && [ "$(ls -A data)" ]; then | |
| echo "scrapers_succeeded=true" >> $GITHUB_OUTPUT | |
| echo "✓ Scrapers completed successfully" | |
| else | |
| echo "scrapers_succeeded=false" >> $GITHUB_OUTPUT | |
| echo "⚠️ No scraper data found" | |
| fi | |
| - name: Commit scraper data | |
| if: steps.scraper-status.outputs.scrapers_succeeded == 'true' | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| # Add only data directory changes | |
| git add data/ | |
| # Check if there are changes to commit | |
| if git diff --staged --quiet; then | |
| echo "No new data to commit" | |
| exit 0 | |
| fi | |
| DATE=$(date +%Y-%m-%d) | |
| SOURCE_COUNT=$(find data -name "*.json" | wc -l | tr -d ' ') | |
| git commit -m "chore: update prices for $DATE ($SOURCE_COUNT sources)" -m "Generated with GitHub Actions" | |
| # Try to push, handle concurrent commits | |
| if git push; then | |
| echo "✓ Push succeeded" | |
| else | |
| echo "⚠️ Push failed, attempting to recover..." | |
| # Stash our commit temporarily | |
| git reset --soft HEAD~1 | |
| git stash | |
| # Pull latest changes | |
| git pull --rebase | |
| # Try to restore our changes | |
| if git stash pop; then | |
| echo "✓ Stash applied successfully, retrying commit..." | |
| # Re-add and commit | |
| git add data/ | |
| if git diff --staged --quiet; then | |
| echo "No new data after merge (likely already committed)" | |
| exit 0 | |
| fi | |
| SOURCE_COUNT=$(find data -name "*.json" | wc -l | tr -d ' ') | |
| git commit -m "chore: update prices for $DATE ($SOURCE_COUNT sources)" -m "Generated with GitHub Actions" | |
| git push | |
| else | |
| echo "✗ Conflict detected - scrape data for this day may already exist" | |
| git stash drop || true | |
| exit 1 | |
| fi | |
| fi | |
| - name: Build frontend | |
| run: npm run build | |
| - name: Upload Pages artifact | |
| uses: actions/upload-pages-artifact@v3 | |
| with: | |
| path: dist | |
| deploy: | |
| needs: scrape-and-build | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deployment.outputs.page_url }} | |
| steps: | |
| - name: Deploy to GitHub Pages | |
| id: deployment | |
| uses: actions/deploy-pages@v4 |