unstructured-api/.github/workflows/ci.yml at 65df0019443e9734e7976c1533b740a0684f19a8 · Unstructured-IO/unstructured-api · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
name: CI

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

env:
  PIPELINE_FAMILY: "general"

jobs:
  lint:
    runs-on: opensource-linux-8core
    steps:
    - uses: actions/checkout@v5
    - name: Read Python version from .python-version
      run: echo "PYTHON_VERSION=$(cat .python-version)" >> $GITHUB_ENV
    - name: Install uv
      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true
        cache-dependency-glob: "uv.lock"
    - name: Set up Python ${{ env.PYTHON_VERSION }}
      run: uv python install ${{ env.PYTHON_VERSION }}
    - name: Install lint dependencies
      run: uv sync --only-group lint --frozen
    - name: Lint
      run: make check

  shellcheck:
    runs-on: opensource-linux-8core
    steps:
      - uses: actions/checkout@v5
      - name: ShellCheck
        uses: ludeeus/action-shellcheck@master

  test:
    runs-on: opensource-linux-8core
    needs: lint
    steps:
    - uses: actions/checkout@v5
    - name: Read Python version from .python-version
      run: echo "PYTHON_VERSION=$(cat .python-version)" >> $GITHUB_ENV
    - name: Install uv
      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true
        cache-dependency-glob: "uv.lock"
    - name: Set up Python ${{ env.PYTHON_VERSION }}
      run: uv python install ${{ env.PYTHON_VERSION }}
    - name: Install dependencies and run core tests
      run: |
        sudo apt-get update && sudo apt-get install --yes poppler-utils libreoffice
        uv sync --group test --frozen
        make install-pandoc
        make install-nltk-models
        sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5
        sudo apt-get install -y tesseract-ocr tesseract-ocr-kor
        tesseract --version
        make test
        make check-coverage

  changelog:
    runs-on: opensource-linux-8core
    steps:
    - uses: actions/checkout@v5
    - if: github.ref != 'refs/heads/main'
      uses: dorny/paths-filter@v4
      id: changes
      with:
        filters: |
          src:
            - 'doc_recipe/**'
            - 'recipe-notebooks/**'

    - if: steps.changes.outputs.src == 'true' && github.ref != 'refs/heads/main'
      uses: dangoslen/changelog-enforcer@v3

  # TODO - figure out best practice for caching docker images
  test_dockerfile:
    runs-on: opensource-linux-8core
    needs: lint
    steps:
    - uses: actions/checkout@v5
    - name: Read Python version from .python-version
      run: echo "PYTHON_VERSION=$(cat .python-version)" >> $GITHUB_ENV
    - name: Install uv
      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true
        cache-dependency-glob: "uv.lock"
    - name: Set up Python ${{ env.PYTHON_VERSION }}
      run: uv python install ${{ env.PYTHON_VERSION }}
    - name: Free up disk space
      run: |
        # Clear some space (https://github.com/actions/runner-images/issues/2840)
        echo "Disk usage before cleanup:"
        df -h

        # Remove unnecessary pre-installed software
        sudo rm -rf /usr/share/dotnet
        sudo rm -rf /opt/ghc
        sudo rm -rf /usr/local/share/boost
        sudo rm -rf /usr/local/lib/android
        sudo rm -rf /opt/hostedtoolcache/CodeQL
        sudo rm -rf /usr/local/.ghcup
        sudo rm -rf /usr/share/swift

        # Clean up docker to ensure we start fresh
        docker system prune -af --volumes

        echo "Disk usage after cleanup:"
        df -h
    - name: Test Dockerfile
      run: |
        uv sync --group test --frozen
        make docker-build
        make docker-test