czap/.github/workflows/scrape.yml at main · honzajavorek/czap · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
name: scrape

on:
  push:
    branches:
      - main
  schedule:
    - cron: "0 4 * * *"  # every day at 4am
  workflow_dispatch:  # allows manual triggering of the workflow

# Allow GITHUB_TOKEN to deploy to GitHub Pages
permissions:
  contents: write
  pages: write
  id-token: write

concurrency:
  group: "pages"
  cancel-in-progress: true

jobs:
  scrape:
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v6

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: '20'

      - name: Install Poetry
        run: pipx install poetry

      - name: Setup Python
        uses: actions/setup-python@v6
        with:
          python-version-file: pyproject.toml
          cache: poetry

      - name: Install Python dependencies
        run: poetry install

      # For some reason the 'overwrite' option for the Scrapy feed exporters
      # doesn't work reliably for me ¯\_(ツ)_/¯
      - name: Clear data file
        run: rm items.json

      - name: Restore Scrapy cache
        uses: actions/cache/restore@v5
        with:
          path: .scrapy
          key: scrapy-cache-${{ github.run_id }}
          restore-keys: |
            scrapy-cache-

      - name: Scrape
        run: poetry run scrapy crawl czap

      - name: Save Scrapy cache
        uses: actions/cache/save@v5
        with:
          path: .scrapy
          key: scrapy-cache-${{ github.run_id }}

      - name: Check that data got exported
        run: "[ -s items.json ] || exit 1"

      - name: Save to Git
        uses: EndBug/add-and-commit@v10
        with:
          add: items.json
          author_name: "scraper"
          author_email: "scraper@honzajavorek.cz"
          message: "update items 📥"

      - name: Move files
        run: |
          mkdir -p site
          mv items.* site/

      - name: Setup Pages
        uses: actions/configure-pages@v6

      - name: Upload artifact
        uses: actions/upload-pages-artifact@v4
        with:
          path: ./site

      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v5