Test

.ci/scripts/wheel/test_wheel.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Cross-platform wheel smoke-test entry point.
+
+The PyTorch Tokenizers wheel should already be installed in the active
+environment when this script runs. We execute the Python unit test suite
+to ensure the wheel exposes the expected bindings and core functionality.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+from typing import Sequence
+
+
+def run_pytest(test_files: Sequence[Path], cwd: Path) -> int:
+    """Execute pytest on the provided files and propagate the return code."""
+    env = os.environ.copy()
+    pythonpath_entries = [str(cwd)]
+    if existing_path := env.get("PYTHONPATH"):
+        pythonpath_entries.append(existing_path)
+    env["PYTHONPATH"] = os.pathsep.join(pythonpath_entries)
+
+    cmd = [
+        sys.executable,
+        "-m",
+        "pytest",
+        "-vv",
+        *(str(test) for test in test_files),
+    ]
+
+    print(f"Running pytest with: {' '.join(cmd)}")
+    result = subprocess.run(cmd, cwd=str(cwd), env=env, check=False)
+    return result.returncode
+
+
+def main() -> int:
+    repo_root = Path(__file__).resolve().parents[3]
+    test_dir = repo_root / "test"
+
+    if not test_dir.exists():
+        print(f"ERROR: Test directory not found: {test_dir}", file=sys.stderr)
+        return 1
+
+    return run_pytest([test_dir], repo_root)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/build-wheels-linux-aarch64.yml

@@ -0,0 +1,52 @@
+name: Nova Build Wheels (Linux AArch64)
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - .ci/**/*
+      - .github/workflows/build-wheels-linux-aarch64.yml
+      - pyproject.toml
+      - setup.py
+    tags:
+      - ciflow/binaries/*
+  push:
+    tags:
+      - release/*
+  tags:
+      # NOTE: Binary build pipelines should only get triggered on release candidate builds
+      # Release candidate tags look like: v1.11.0-rc1
+      - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
+      - ciflow/binaries/*
+  release:
+    types: [published]
+
+jobs:
+  generate-matrix:
+    name: Generate Linux AArch64 matrix
+    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
+    with:
+      package-type: wheel
+      os: linux-aarch64
+      test-infra-repository: pytorch/test-infra
+      test-infra-ref: main
+      with-cuda: disable
+      with-rocm: disable
+      with-cpu: enable
+      with-xpu: disable
+      python-versions: ${{ env.PYTHON_VERSIONS_JSON }}
+
+  build:
+    name: Build Linux AArch64 wheels
+    needs: generate-matrix
+    uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
+    with:
+      repository: meta-pytorch/tokenizers
+      ref: ${{ github.event_name == 'release' && github.event.release.tag_name || github.ref_name }}
+      test-infra-repository: pytorch/test-infra
+      test-infra-ref: main
+      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
+      trigger-event: ${{ github.event_name }}
+      package-name: pytorch_tokenizers
+      smoke-test-script: .ci/scripts/wheel/test_wheel.py
+      architecture: aarch64

.github/workflows/build-wheels-linux.yml

@@ -0,0 +1,51 @@
+name: Nova Build Wheels (Linux x86_64)
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - .ci/**/*
+      - .github/workflows/build-wheels-windows.yml
+      - pyproject.toml
+      - setup.py
+    tags:
+      - ciflow/binaries/*
+  push:
+    tags:
+      - release/*
+  tags:
+      # NOTE: Binary build pipelines should only get triggered on release candidate builds
+      # Release candidate tags look like: v1.11.0-rc1
+      - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
+      - ciflow/binaries/*
+  release:
+    types: [published]
+
+jobs:
+  generate-matrix:
+    name: Generate Linux matrix
+    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
+    with:
+      package-type: wheel
+      os: linux
+      test-infra-repository: pytorch/test-infra
+      test-infra-ref: main
+      with-cuda: disable
+      with-rocm: disable
+      with-cpu: enable
+      with-xpu: disable
+      python-versions: ${{ env.PYTHON_VERSIONS_JSON }}
+
+  build:
+    name: Build Linux wheels
+    needs: generate-matrix
+    uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
+    with:
+      repository: meta-pytorch/tokenizers
+      ref: ${{ github.event_name == 'release' && github.event.release.tag_name || github.ref_name }}
+      test-infra-repository: pytorch/test-infra
+      test-infra-ref: main
+      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
+      trigger-event: ${{ github.event_name }}
+      package-name: pytorch_tokenizers
+      smoke-test-script: .ci/scripts/wheel/test_wheel.py

.github/workflows/build-wheels-macos-arm64.yml

@@ -0,0 +1,51 @@
+name: Nova Build Wheels (macOS arm64)
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - .ci/**/*
+      - .github/workflows/build-wheels-macos-arm64.yml
+      - pyproject.toml
+      - setup.py
+    tags:
+      - ciflow/binaries/*
+  push:
+    tags:
+      - release/*
+  tags:
+      # NOTE: Binary build pipelines should only get triggered on release candidate builds
+      # Release candidate tags look like: v1.11.0-rc1
+      - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
+      - ciflow/binaries/*
+  release:
+    types: [published]
+
+jobs:
+  generate-matrix:
+    name: Generate macOS matrix
+    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
+    with:
+      package-type: wheel
+      os: macos-arm64
+      test-infra-repository: pytorch/test-infra
+      test-infra-ref: main
+      with-cuda: disable
+      with-rocm: disable
+      with-cpu: enable
+      with-xpu: disable
+      python-versions: ${{ env.PYTHON_VERSIONS_JSON }}
+
+  build:
+    name: Build macOS wheels
+    needs: generate-matrix
+    uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main
+    with:
+      repository: meta-pytorch/tokenizers
+      ref: ${{ github.event_name == 'release' && github.event.release.tag_name || github.ref_name }}
+      test-infra-repository: pytorch/test-infra
+      test-infra-ref: main
+      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
+      trigger-event: ${{ github.event_name }}
+      package-name: pytorch_tokenizers
+      smoke-test-script: .ci/scripts/wheel/test_wheel.py

.github/workflows/build-wheels-windows.yml

@@ -0,0 +1,51 @@
+name: Nova Build Wheels (Windows)
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - .ci/**/*
+      - .github/workflows/build-wheels-windows.yml
+      - pyproject.toml
+      - setup.py
+    tags:
+      - ciflow/binaries/*
+  push:
+    tags:
+      - release/*
+  tags:
+      # NOTE: Binary build pipelines should only get triggered on release candidate builds
+      # Release candidate tags look like: v1.11.0-rc1
+      - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
+      - ciflow/binaries/*
+  release:
+    types: [published]
+
+jobs:
+  generate-matrix:
+    name: Generate Windows matrix
+    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
+    with:
+      package-type: wheel
+      os: windows
+      test-infra-repository: pytorch/test-infra
+      test-infra-ref: main
+      with-cuda: disable
+      with-rocm: disable
+      with-cpu: enable
+      with-xpu: disable
+      python-versions: ${{ env.PYTHON_VERSIONS_JSON }}
+
+  build:
+    name: Build Windows wheels
+    needs: generate-matrix
+    uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main
+    with:
+      repository: meta-pytorch/tokenizers
+      ref: ${{ github.event_name == 'release' && github.event.release.tag_name || github.ref_name }}
+      test-infra-repository: pytorch/test-infra
+      test-infra-ref: main
+      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
+      trigger-event: ${{ github.event_name }}
+      package-name: pytorch_tokenizers
+      smoke-test-script: .ci/scripts/wheel/test_wheel.py