opensanctions/.github/workflows/build.yml at main · kdeden/opensanctions · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
name: build

on:
  - push
  - pull_request
  - workflow_dispatch

permissions:
  packages: write
  id-token: write


jobs:
  python:
    env:
      OPENSANCTIONS_RESOLVER_PATH: /dev/null
      ZAVOD_SYNC_POSITIONS: false

    runs-on: ubuntu-latest
    strategy:
      matrix:
        python:
          - "3.10"
          - "3.11"
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
      - name: Install system dependencies
        env:
          DEBIAN_FRONTEND: noninteractive
        run: |
          sudo apt-get install -y -qq libicu-dev poppler-utils
      - name: Install zavod dependencies
        working-directory: zavod
        run: |
          pip install --no-cache-dir -q -e ".[dev]"
          pip freeze
      - name: Check zavod type annotations (strict)
        working-directory: zavod
        run: |
          make typecheck
      - name: Check zavod ruff lint
        working-directory: zavod
        run: |
          make lint
      - name: Run zavod tests
        working-directory: zavod
        run: |
          make test
      - name: Build a zavod distribution
        working-directory: zavod
        run: |
          python setup.py sdist bdist_wheel
      - name: Get any modified dataset files
        id: changed-files
        uses: tj-actions/changed-files@v44
        with:
          files: "datasets/**"
      - name: Crawl modified datasets
        if: ${{ steps.changed-files.outputs.all_changed_files != '' }}
        run: |
          set -euo pipefail
          datasets=$(python contrib/ci_datasets.py ${{ steps.changed-files.outputs.all_changed_files }})
          echo "$datasets"
          for dataset in $datasets
          do
              echo Crawling $dataset
              zavod crawl -c $dataset
          done
      - name: Validate modified datasets
        if: ${{ steps.changed-files.outputs.all_changed_files != '' }}
        run: |
          set -euo pipefail
          datasets=$(python contrib/ci_datasets.py ${{ steps.changed-files.outputs.all_changed_files }})
          for dataset in $datasets
          do
              echo Validating $dataset
              zavod validate -c $dataset
          done
      - name: Export modified datasets
        if: ${{ steps.changed-files.outputs.all_changed_files != '' }}
        run: |
          set -euo pipefail
          datasets=$(python contrib/ci_datasets.py ${{ steps.changed-files.outputs.all_changed_files }})
          for dataset in $datasets
          do
              echo Exporting $dataset
              zavod export -c $dataset
          done
      - name: Publish a Python distribution to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        # working-directory: zavod
        if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/zavod')
        with:
          skip-existing: true
          packages-dir: zavod/dist

  docker:
    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
    runs-on: ubuntu-latest
    needs: [python]
    steps:
      - uses: actions/checkout@v4
      # - name: Set up QEMU
      #   uses: docker/setup-qemu-action@v2
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
        with:
          install: true
      - name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Build and push release
        uses: docker/build-push-action@v6
        with:
          context: .
          pull: true
          push: true
          tags: ghcr.io/opensanctions/opensanctions:latest
          cache-from: type=gha
          cache-to: type=gha,mode=max

  dispatch:
    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
    runs-on: ubuntu-latest
    needs: [docker, python]
    steps:
      - name: Dispatch add-on build
        uses: peter-evans/repository-dispatch@v3
        with:
          token: ${{ secrets.OPERATIONS_PAT }}
          repository: opensanctions/operations
          event-type: build-etl