scrape-and-deploy.yml

name: Scrape Prices and Deploy

on:
  # Run daily at midnight UTC
  schedule:
    - cron: '0 0 * * *'

  # Allow manual trigger for testing
  workflow_dispatch:

permissions:
  contents: write
  pages: write
  id-token: write

jobs:
  scrape-and-build:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'
          cache: 'npm'

      - name: Install dependencies
        run: npm ci

      - name: Install Chromium for Puppeteer
        run: |
          npx puppeteer browsers install chrome

      - name: Run all scrapers
        run: |
          node scrapers/run-all.js
        continue-on-error: true
        env:
          # Run in CI mode (headless, no sandbox)
          CI: true

      - name: Check scraper results
        id: scraper-status
        run: |
          if [ -d "data" ] && [ "$(ls -A data)" ]; then
            echo "scrapers_succeeded=true" >> $GITHUB_OUTPUT
            echo "✓ Scrapers completed successfully"
          else
            echo "scrapers_succeeded=false" >> $GITHUB_OUTPUT
            echo "⚠️ No scraper data found"
          fi

      - name: Commit scraper data
        if: steps.scraper-status.outputs.scrapers_succeeded == 'true'
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"

          # Add only data directory changes
          git add data/

          # Check if there are changes to commit
          if git diff --staged --quiet; then
            echo "No new data to commit"
            exit 0
          fi

          DATE=$(date +%Y-%m-%d)
          SOURCE_COUNT=$(find data -name "*.json" | wc -l | tr -d ' ')
          git commit -m "chore: update prices for $DATE ($SOURCE_COUNT sources)" -m "Generated with GitHub Actions"

          # Try to push, handle concurrent commits
          if git push; then
            echo "✓ Push succeeded"
          else
            echo "⚠️ Push failed, attempting to recover..."

            # Stash our commit temporarily
            git reset --soft HEAD~1
            git stash

            # Pull latest changes
            git pull --rebase

            # Try to restore our changes
            if git stash pop; then
              echo "✓ Stash applied successfully, retrying commit..."

              # Re-add and commit
              git add data/
              if git diff --staged --quiet; then
                echo "No new data after merge (likely already committed)"
                exit 0
              fi

              SOURCE_COUNT=$(find data -name "*.json" | wc -l | tr -d ' ')
              git commit -m "chore: update prices for $DATE ($SOURCE_COUNT sources)" -m "Generated with GitHub Actions"
              git push
            else
              echo "✗ Conflict detected - scrape data for this day may already exist"
              git stash drop || true
              exit 1
            fi
          fi

      - name: Build frontend
        run: npm run build

      - name: Upload Pages artifact
        uses: actions/upload-pages-artifact@v3
        with:
          path: dist

  deploy:
    needs: scrape-and-build
    runs-on: ubuntu-latest

    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}

    steps:
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v4