diff --git a/.ci/scripts/wheel/test_wheel.py b/.ci/scripts/wheel/test_wheel.py new file mode 100644 index 0000000..9733fd4 --- /dev/null +++ b/.ci/scripts/wheel/test_wheel.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +""" +Cross-platform wheel smoke-test entry point. + +The PyTorch Tokenizers wheel should already be installed in the active +environment when this script runs. We execute the Python unit test suite +to ensure the wheel exposes the expected bindings and core functionality. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path +from typing import Sequence + + +def run_pytest(test_files: Sequence[Path], cwd: Path) -> int: + """Execute pytest on the provided files and propagate the return code.""" + env = os.environ.copy() + pythonpath_entries = [str(cwd)] + if existing_path := env.get("PYTHONPATH"): + pythonpath_entries.append(existing_path) + env["PYTHONPATH"] = os.pathsep.join(pythonpath_entries) + + cmd = [ + sys.executable, + "-m", + "pytest", + "-vv", + *(str(test) for test in test_files), + ] + + print(f"Running pytest with: {' '.join(cmd)}") + result = subprocess.run(cmd, cwd=str(cwd), env=env, check=False) + return result.returncode + + +def main() -> int: + repo_root = Path(__file__).resolve().parents[3] + test_dir = repo_root / "test" + + if not test_dir.exists(): + print(f"ERROR: Test directory not found: {test_dir}", file=sys.stderr) + return 1 + + return run_pytest([test_dir], repo_root) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/build-wheels-linux-aarch64.yml b/.github/workflows/build-wheels-linux-aarch64.yml new file mode 100644 index 0000000..768cacb --- /dev/null +++ b/.github/workflows/build-wheels-linux-aarch64.yml @@ -0,0 +1,52 @@ +name: Nova Build Wheels (Linux AArch64) + +on: + workflow_dispatch: + pull_request: + paths: + - .ci/**/* + - .github/workflows/build-wheels-linux-aarch64.yml + - pyproject.toml + - setup.py + tags: + - ciflow/binaries/* + push: + tags: + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + - ciflow/binaries/* + release: + types: [published] + +jobs: + generate-matrix: + name: Generate Linux AArch64 matrix + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux-aarch64 + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-cuda: disable + with-rocm: disable + with-cpu: enable + with-xpu: disable + python-versions: ${{ env.PYTHON_VERSIONS_JSON }} + + build: + name: Build Linux AArch64 wheels + needs: generate-matrix + uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main + with: + repository: meta-pytorch/tokenizers + ref: ${{ github.event_name == 'release' && github.event.release.tag_name || github.ref_name }} + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + trigger-event: ${{ github.event_name }} + package-name: pytorch_tokenizers + smoke-test-script: .ci/scripts/wheel/test_wheel.py + architecture: aarch64 diff --git a/.github/workflows/build-wheels-linux.yml b/.github/workflows/build-wheels-linux.yml new file mode 100644 index 0000000..764d063 --- /dev/null +++ b/.github/workflows/build-wheels-linux.yml @@ -0,0 +1,51 @@ +name: Nova Build Wheels (Linux x86_64) + +on: + workflow_dispatch: + pull_request: + paths: + - .ci/**/* + - .github/workflows/build-wheels-windows.yml + - pyproject.toml + - setup.py + tags: + - ciflow/binaries/* + push: + tags: + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + - ciflow/binaries/* + release: + types: [published] + +jobs: + generate-matrix: + name: Generate Linux matrix + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-cuda: disable + with-rocm: disable + with-cpu: enable + with-xpu: disable + python-versions: ${{ env.PYTHON_VERSIONS_JSON }} + + build: + name: Build Linux wheels + needs: generate-matrix + uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main + with: + repository: meta-pytorch/tokenizers + ref: ${{ github.event_name == 'release' && github.event.release.tag_name || github.ref_name }} + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + trigger-event: ${{ github.event_name }} + package-name: pytorch_tokenizers + smoke-test-script: .ci/scripts/wheel/test_wheel.py diff --git a/.github/workflows/build-wheels-macos-arm64.yml b/.github/workflows/build-wheels-macos-arm64.yml new file mode 100644 index 0000000..c7f0a05 --- /dev/null +++ b/.github/workflows/build-wheels-macos-arm64.yml @@ -0,0 +1,51 @@ +name: Nova Build Wheels (macOS arm64) + +on: + workflow_dispatch: + pull_request: + paths: + - .ci/**/* + - .github/workflows/build-wheels-macos-arm64.yml + - pyproject.toml + - setup.py + tags: + - ciflow/binaries/* + push: + tags: + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + - ciflow/binaries/* + release: + types: [published] + +jobs: + generate-matrix: + name: Generate macOS matrix + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: macos-arm64 + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-cuda: disable + with-rocm: disable + with-cpu: enable + with-xpu: disable + python-versions: ${{ env.PYTHON_VERSIONS_JSON }} + + build: + name: Build macOS wheels + needs: generate-matrix + uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main + with: + repository: meta-pytorch/tokenizers + ref: ${{ github.event_name == 'release' && github.event.release.tag_name || github.ref_name }} + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + trigger-event: ${{ github.event_name }} + package-name: pytorch_tokenizers + smoke-test-script: .ci/scripts/wheel/test_wheel.py diff --git a/.github/workflows/build-wheels-windows.yml b/.github/workflows/build-wheels-windows.yml new file mode 100644 index 0000000..4cfca92 --- /dev/null +++ b/.github/workflows/build-wheels-windows.yml @@ -0,0 +1,51 @@ +name: Nova Build Wheels (Windows) + +on: + workflow_dispatch: + pull_request: + paths: + - .ci/**/* + - .github/workflows/build-wheels-windows.yml + - pyproject.toml + - setup.py + tags: + - ciflow/binaries/* + push: + tags: + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + - ciflow/binaries/* + release: + types: [published] + +jobs: + generate-matrix: + name: Generate Windows matrix + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: windows + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-cuda: disable + with-rocm: disable + with-cpu: enable + with-xpu: disable + python-versions: ${{ env.PYTHON_VERSIONS_JSON }} + + build: + name: Build Windows wheels + needs: generate-matrix + uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main + with: + repository: meta-pytorch/tokenizers + ref: ${{ github.event_name == 'release' && github.event.release.tag_name || github.ref_name }} + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + trigger-event: ${{ github.event_name }} + package-name: pytorch_tokenizers + smoke-test-script: .ci/scripts/wheel/test_wheel.py