tokenizers/.github/workflows/python.yml at 387b2183ff1857de4fa8ba5e23816718437e7a89 · huggingface/tokenizers · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
name: Python

on:
  push:
    branches:
      - main
    paths-ignore:
      - bindings/node/**
  pull_request:
    paths-ignore:
      - bindings/node/**

jobs:
  build_win_32:
    name: Check it builds for Windows 32-bit
    runs-on: windows-latest
    strategy:
      matrix:
        python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Install Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable-i686-pc-windows-msvc
          override: true

      - name: Override toolchain
        shell: bash
        working-directory: ./bindings/python
        run: echo "stable-i686-pc-windows-msvc" > rust-toolchain


      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
          architecture: x86

      - name: Build
        uses: actions-rs/cargo@v1
        with:
          command: build
          args: --manifest-path ./bindings/python/Cargo.toml


  build_and_test:
    name: Check everything builds & tests
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, macos-latest]
        python: ["3.14", "3.14t"]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4


      - name: Install Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          components: rustfmt, clippy

      - name: Install audit
        uses: actions-rs/cargo@v1
        with:
          command: install
          args: cargo-audit

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
          architecture: "x64"


      - name: Cache Cargo Registry
        uses: actions/cache@v4
        with:
          path: ~/.cargo/registry
          key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}

          # - name: Cache Cargo Build Target
          #   uses: actions/cache@v1
          #   with:
          #     path: ./bindings/python/target
          #     key: ${{ runner.os }}-cargo-python-build-${{ hashFiles('**/Cargo.lock') }}

      - name: Lint with RustFmt
        uses: actions-rs/cargo@v1
        with:
          toolchain: stable
          command: fmt
          args: --manifest-path ./bindings/python/Cargo.toml -- --check

      - name: Lint with Clippy
        uses: actions-rs/cargo@v1
        with:
          command: clippy
          args: --manifest-path ./bindings/python/Cargo.toml --all-targets --all-features -- -D warnings

      - name: Install cargo-audit
        run: cargo install cargo-audit

      - name: Run Audit
        uses: actions-rs/cargo@v1
        with:
          command: audit
          args: -D warnings -f ./bindings/python/Cargo.lock  --ignore RUSTSEC-2024-0436 --ignore RUSTSEC-2025-0014 --ignore RUSTSEC-2025-0119 --ignore RUSTSEC-2024-0436

      - name: Install
        working-directory: ./bindings/python
        run: |
          python -m venv .env
          source .env/bin/activate
          pip install -U pip
          pip install pytest requests setuptools_rust numpy pyarrow datasets ty
          pip install -e .[dev]

      - name: Check style
        working-directory: ./bindings/python
        run: |
          source .env/bin/activate
          make check-style

      - name: Type check
        working-directory: ./bindings/python
        run: |
          source .env/bin/activate
          ty check py_src --exclude py_src/tokenizers/implementations

      - name: Run tests
        working-directory: ./bindings/python
        run: |
          source .env/bin/activate
          make test